import cv2
import gradio as gr
import tempfile
import torch
from torchvision.models.detection import fasterrcnn_resnet50_fpn
import torchvision.transforms as transforms
from PIL import Image
import numpy as np
import soundfile as sf

class FasterRCNNDetector:
    def __init__(self):
        self.model = fasterrcnn_resnet50_fpn(pretrained=True)
        self.model.eval()
        self.classes = [
            "__background__", "person", "bicycle", "car", "motorcycle", "airplane", "bus",
            "train", "truck", "boat", "traffic light", "fire hydrant", "N/A", "stop sign",
            "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
            "elephant", "bear", "zebra", "giraffe", "N/A", "backpack", "umbrella", "N/A", "N/A",
            "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball",
            "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket",
            "bottle", "N/A", "wine glass", "cup", "fork", "knife", "spoon", "bowl",
            "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza",
            "donut", "cake", "chair", "couch", "potted plant", "bed", "N/A", "dining table",
            "N/A", "N/A", "toilet", "N/A", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
            "microwave", "oven", "toaster", "sink", "refrigerator", "N/A", "book",
            "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"
        ]

    def detect_objects(self, image):
        image_pil = Image.fromarray(image)
        transform = transforms.Compose([transforms.ToTensor()])
        image_tensor = transform(image_pil).unsqueeze(0)
        
        with torch.no_grad():
            prediction = self.model(image_tensor)
        
        boxes = prediction[0]['boxes']
        labels = prediction[0]['labels']
        scores = prediction[0]['scores']
        
        for box, label, score in zip(boxes, labels, scores):
            box = [int(i) for i in box]
            cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2)
            cv2.putText(image, self.classes[label], (box[0], box[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (36,255,12), 2)
        
        return image

class JarvisModels:
    def __init__(self):
        self.client1 = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
        self.model = deepspeech.Model("deepspeech-0.9.3-models.pbmm")
        self.model.setBeamWidth(500)

    async def generate_response(self, prompt):
        generate_kwargs = dict(
            temperature=0.6,
            max_new_tokens=256,
            top_p=0.95,
            repetition_penalty=1,
            do_sample=True,
            seed=42,
        )
        formatted_prompt = system_instructions1 + prompt + "[JARVIS]"
        stream = self.client1.text_generation(
            formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=True)
        output = ""
        for response in stream:
            output += response.token.text

        communicate = edge_tts.Communicate(output)
        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
            tmp_path = tmp_file.name
            communicate.save(tmp_path)
        return tmp_path

def transcribe_audio(audio_file):
    model = JarvisModels().model
    audio, sample_rate = sf.read(audio_file)
    return model.stt(audio)

def generate_response(frame):
    jarvis = JarvisModels()
    response_model = await jarvis.generate_response("Hello, I see some interesting objects!")
    return response_model

detector = FasterRCNNDetector()

iface = gr.Interface(
    fn=[detector.detect_objects, transcribe_audio],
    inputs=gr.inputs.Video(label="Webcam", parameters={"fps": 30}),
    outputs=[gr.outputs.Image(), "text"],
    title="Vision and Speech Interface",
    description="This interface detects objects in the webcam feed and transcribes speech recorded through the microphone."
)
iface.launch()