Update crew.py

crew.py

@@ -4,12 +4,12 @@ from crewai.tools import tool
 from crewai_tools import (
     CodeInterpreterTool,
     SerperDevTool,
-    WebsiteSearchTool,
-    YoutubeVideoSearchTool
+    WebsiteSearchTool
 )
 from openai import OpenAI
 from openinference.instrumentation.crewai import CrewAIInstrumentor
 from phoenix.otel import register
+from pytube import YouTube
 from util import get_final_answer, get_img_b64
 
 ## LLMs
@@ -38,6 +38,43 @@ CrewAIInstrumentor().instrument(tracer_provider=tracer_provider)
 def run_crew(question, file_path):
     # Custom tools
 
+    @tool("Image Analysis Tool")
+    def image_analysis_tool(question: str, file_path: str) -> str:
+        """Answer a question about an image file.
+    
+           Args:
+               question (str): Question about the image file
+               file_path (str): Path of the image file
+                
+           Returns:
+               str: Answer to the question about the image file
+                
+           Raises:
+               FileNotFoundError: If the image file does not exist
+               RuntimeError: If processing fails"""
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"Image file not found: {file_path}")
+            
+        try:
+            # Get image
+            
+            img_b64 = get_img_b64(file_path)
+
+            # OpenAI
+            
+            client = OpenAI()
+
+            completion = client.chat.completions.create(
+                messages=[{"role": "user",
+                           "content": [{"type": "text", "text": question},
+                                       {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}]}],
+                model=IMAGE_MODEL
+            )
+        
+            return completion.choices[0].message.content
+        except Exception as e:
+            raise RuntimeError(f"Failed to process image: {str(e)}")
+
     @tool("Audio Analysis Tool")
     def audio_analysis_tool(question: str, file_path: str) -> str:
         """Answer a question about an audio file.
@@ -67,59 +104,72 @@ def run_crew(question, file_path):
             return transcript.text
         except Exception as e:
             raise RuntimeError(f"Failed to process audio: {str(e)}")
-
-    @tool("Image Analysis Tool")
-    def image_analysis_tool(question: str, file_path: str) -> str:
-        """Answer a question about an image file.
+    
+    @tool("YouTube Audio Analysis Tool")
+    def youtube_audio_analysis_tool(question: str, url: str) -> str:
+        """Answer a question about YouTube audio.
     
            Args:
-               question (str): Question about the image file
-               file_path (str): Path of the image file
+               question (str): Question about YouTube audio
+               url (str): YouTube URL
                 
            Returns:
-               str: Answer to the question about the image file
+               str: Answer to the question about YouTube audio
                 
            Raises:
-               FileNotFoundError: If the image file does not exist
                RuntimeError: If processing fails"""
-        if not os.path.exists(file_path):
-            raise FileNotFoundError(f"Image file not found: {file_path}")
+        try:
+            # YouTube
             
-        try:  
-            img_b64 = get_img_b64(file_path)
+            file = "audio.mp4"
+            
+            yt = YouTube(url)
+            
+            stream = yt.streams.filter(only_audio=True).first()
+            
+            stream.download(output_path=".", filename=file)
 
+            # OpenAI
+            
             client = OpenAI()
 
-            completion = client.chat.completions.create(
-                messages=[{"role": "user",
-                           "content": [{"type": "text", "text": question},
-                                       {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}]}],
-                model=IMAGE_MODEL
+            transcription = client.audio.transcriptions.create(
+                file=open(file, "rb"),
+                model=AUDIO_MODEL,
+                prompt=question
             )
-        
-            return completion.choices[0].message.content
+            
+            return transcription.text
         except Exception as e:
-            raise RuntimeError(f"Failed to process image: {str(e)}")
+            raise RuntimeError(f"Failed to process video: {str(e)}")
 
-    @tool("Video Analysis Tool")
-    def video_analysis_tool(question: str, file_path: str) -> str:
-        """Answer a question about a video file.
+    @tool("YouTube Video Analysis Tool")
+    def youtube_video_analysis_tool(question: str, url: str) -> str:
+        """Answer a question about YouTube video.
     
            Args:
-               question (str): Question about the video file
-               file_path (str): Path of the video file
+               question (str): Question about YouTube video
+               url (str): YouTube URL
                 
            Returns:
-               str: Answer to the question about the video file
+               str: Answer to the question about YouTube video
                 
            Raises:
-               FileNotFoundError: If the video file does not exist
                RuntimeError: If processing fails"""
-        if not os.path.exists(file_path):
-            raise FileNotFoundError(f"Video file not found: {file_path}")
-            
         try:
-            video = cv2.VideoCapture(file_path)
+            # YouTube
+            
+            file = "video.mp4"
+            
+            yt = YouTube(url)
+            
+            stream = yt.streams.get_highest_resolution()
+            
+            stream.download(output_path=".", filename=file)
+
+            # Get video TODOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOo
+            
+            video = cv2.VideoCapture(file)
             
             base64Frames = []
             
@@ -134,7 +184,11 @@ def run_crew(question, file_path):
                 base64Frames.append(base64.b64encode(buffer).decode("utf-8"))
             
             video.release()
+            
+            # OpenAI
 
+            client = OpenAI()
+            
             response = client.responses.create(
                 input=[{"role": "user",
                         "content": [{"type": "input_text", "text": (question)},
@@ -148,24 +202,23 @@ def run_crew(question, file_path):
     
     # Built-in tools
 
-    python_coding_tool = CodeInterpreterTool()
     web_search_tool = SerperDevTool()
     web_rag_tool = WebsiteSearchTool()
-    youtube_video_analysis_tool = YoutubeVideoSearchTool()
+    python_coding_tool = CodeInterpreterTool()
 
     # Agents
 
-    audio_analysis_agent = Agent(
-        role="Audio Analysis Agent",
-        goal="Analyze audio to help answer question \"{question}\"",
-        backstory="As an expert audio analysis assistant, you analyze the audio to help answer the question.",
+    web_search_agent = Agent(
+        role="Web Search Agent",
+        goal="Search the web to help answer question \"{question}\", then scrape the most relevant web page.",
+        backstory="As an expert web search assistant, you search the web to help answer the question.",
         allow_delegation=False,
         llm=AGENT_MODEL,
         max_iter=3,
-        tools=[audio_analysis_tool],
+        tools=[web_search_tool, web_rag_tool],
         verbose=False
     )
-    
+
     image_analysis_agent = Agent(
         role="Image Analysis Agent",
         goal="Analyze image to help answer question \"{question}\"",
@@ -176,40 +229,29 @@ def run_crew(question, file_path):
         tools=[image_analysis_tool],
         verbose=False
     )
-    
-    python_coding_agent = Agent(
-        role="Python Coding Agent",
-        goal="Write and/or execute Python code to help answer question \"{question}\"",
-        backstory="As an expert Python coding assistant, you write and/or execute Python code to help answer the question.",
-        allow_delegation=False,
-        llm=AGENT_MODEL,
-        max_iter=10,
-        tools=[python_coding_tool],
-        verbose=False
-    )
 
-    video_analysis_agent = Agent(
-        role="Video Analysis Agent",
-        goal="Analyze video to help answer question \"{question}\"",
-        backstory="As an expert video analysis assistant, you analyze the video to help answer the question.",
+    audio_analysis_agent = Agent(
+        role="Audio Analysis Agent",
+        goal="Analyze audio to help answer question \"{question}\"",
+        backstory="As an expert audio analysis assistant, you analyze the audio to help answer the question.",
         allow_delegation=False,
         llm=AGENT_MODEL,
         max_iter=3,
-        tools=[video_analysis_tool],
+        tools=[audio_analysis_tool],
         verbose=False
     )
 
-    web_search_agent = Agent(
-        role="Web Search Agent",
-        goal="Search the web to help answer question \"{question}\", then scrape the most relevant web page.",
-        backstory="As an expert web search assistant, you search the web to help answer the question.",
+    youtube_audio_analysis_agent = Agent(
+        role="YouTube Audio Analysis Agent",
+        goal="Analyze YouTube audio to help answer question \"{question}\"",
+        backstory="As an expert YouTube audio analysis assistant, you analyze the audio to help answer the question.",
         allow_delegation=False,
         llm=AGENT_MODEL,
         max_iter=3,
-        tools=[web_search_tool, web_rag_tool],
+        tools=[video_analysis_tool],
         verbose=False
     )
-
+    
     youtube_video_analysis_agent = Agent(
         role="YouTube Video Analysis Agent",
         goal="Analyze YouTube video to help answer question \"{question}\"",
@@ -217,10 +259,21 @@ def run_crew(question, file_path):
         allow_delegation=False,
         llm=AGENT_MODEL,
         max_iter=3,
-        tools=[youtube_video_analysis_tool],
+        tools=[video_analysis_tool],
         verbose=False
     )
-    
+
+    python_coding_agent = Agent(
+        role="Python Coding Agent",
+        goal="Write and/or execute Python code to help answer question \"{question}\"",
+        backstory="As an expert Python coding assistant, you write and/or execute Python code to help answer the question.",
+        allow_delegation=False,
+        llm=AGENT_MODEL,
+        max_iter=10,
+        tools=[python_coding_tool],
+        verbose=False
+    )
+
     manager_agent = Agent(
         role="Manager Agent",
         goal="Try to answer the following question. If needed, delegate to one or more of your coworkers for help. "
@@ -244,12 +297,12 @@ def run_crew(question, file_path):
     # Crew
     
     crew = Crew(
-        agents=[audio_analysis_agent, 
+        agents=[web_search_agent, 
                 image_analysis_agent, 
-                python_coding_agent, 
-                video_analysis_agent, 
-                web_search_agent, 
-                youtube_video_analysis_agent],
+                audio_analysis_agent, 
+                youtube_audio_analysis_agent, 
+                youtube_video_analysis_agent, 
+                python_coding_agent],
         manager_agent=manager_agent,
         tasks=[manager_task],
         verbose=True