Update app.py

app.py

@@ -5,14 +5,14 @@ from threading import Thread
 import time
 import torch
 import spaces
+import cv2
+import numpy as np
+from PIL import Image
 
-# -----------------------
-# Progress Bar Helper
-# -----------------------
 def progress_bar_html(label: str) -> str:
     """
     Returns an HTML snippet for a thin progress bar with a label.
-    The progress bar is styled as a dark red animated bar.
+    The progress bar is styled as a dark animated bar.
     """
     return f'''
 <div style="display: flex; align-items: center;">
@@ -29,7 +29,32 @@ def progress_bar_html(label: str) -> str:
 </style>
     '''
 
-MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct" #else ; MODEL_ID = "Qwen/Qwen2.5-VL-7B-Instruct"
+def downsample_video(video_path):
+    """
+    Downsamples the video to 10 evenly spaced frames.
+    Each frame is converted to a PIL Image along with its timestamp.
+    """
+    vidcap = cv2.VideoCapture(video_path)
+    total_frames = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
+    fps = vidcap.get(cv2.CAP_PROP_FPS)
+    frames = []
+    if total_frames <= 0 or fps <= 0:
+        vidcap.release()
+        return frames
+    # Sample 10 evenly spaced frames.
+    frame_indices = np.linspace(0, total_frames - 1, 10, dtype=int)
+    for i in frame_indices:
+        vidcap.set(cv2.CAP_PROP_POS_FRAMES, i)
+        success, image = vidcap.read()
+        if success:
+            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+            pil_image = Image.fromarray(image)
+            timestamp = round(i / fps, 2)
+            frames.append((pil_image, timestamp))
+    vidcap.release()
+    return frames
+
+MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"  # Alternatively: "Qwen/Qwen2.5-VL-7B-Instruct"
 processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
 model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
     MODEL_ID,
@@ -42,7 +67,52 @@ def model_inference(input_dict, history):
     text = input_dict["text"]
     files = input_dict["files"]
 
-    # Load images if provided
+    if text.strip().lower().startswith("@video-infer"):
+        # Remove the tag from the query.
+        text = text[len("@video-infer"):].strip()
+        if not files:
+            gr.Error("Please upload a video file along with your @video-infer query.")
+            return
+        # Assume the first file is a video.
+        video_path = files[0]
+        frames = downsample_video(video_path)
+        if not frames:
+            gr.Error("Could not process video.")
+            return
+        # Build messages: start with the text prompt.
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": text}]
+            }
+        ]
+        # Append each frame with a timestamp label.
+        for image, timestamp in frames:
+            messages[0]["content"].append({"type": "text", "text": f"Frame {timestamp}:"})
+            messages[0]["content"].append({"type": "image", "image": image})
+        # Collect only the images from the frames.
+        video_images = [image for image, _ in frames]
+        # Prepare the prompt.
+        prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        inputs = processor(
+            text=[prompt],
+            images=video_images,
+            return_tensors="pt",
+            padding=True,
+        ).to("cuda")
+        # Set up streaming generation.
+        streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
+        generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024)
+        thread = Thread(target=model.generate, kwargs=generation_kwargs)
+        thread.start()
+        buffer = ""
+        yield progress_bar_html("Processing video with Qwen2.5VL Model")
+        for new_text in streamer:
+            buffer += new_text
+            time.sleep(0.01)
+            yield buffer
+        return
+
     if len(files) > 1:
         images = [load_image(image) for image in files]
     elif len(files) == 1:
@@ -50,7 +120,6 @@ def model_inference(input_dict, history):
     else:
         images = []
 
-    # Validate input
     if text == "" and not images:
         gr.Error("Please input a query and optionally image(s).")
         return
@@ -58,7 +127,6 @@ def model_inference(input_dict, history):
         gr.Error("Please input a text query along with the image(s).")
         return
 
-    # Prepare messages for the model
     messages = [
         {
             "role": "user",
@@ -68,8 +136,6 @@ def model_inference(input_dict, history):
             ],
         }
     ]
-
-    # Apply chat template and process inputs
     prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     inputs = processor(
         text=[prompt],
@@ -77,16 +143,10 @@ def model_inference(input_dict, history):
         return_tensors="pt",
         padding=True,
     ).to("cuda")
-
-    # Set up streamer for real-time output
     streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
     generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024)
-
-    # Start generation in a separate thread
     thread = Thread(target=model.generate, kwargs=generation_kwargs)
     thread.start()
-
-    # Stream the output
     buffer = ""
     yield progress_bar_html("Processing with Qwen2.5VL Model")
     for new_text in streamer:
@@ -94,21 +154,19 @@ def model_inference(input_dict, history):
         time.sleep(0.01)
         yield buffer
 
-
-# Example inputs
 examples = [
     [{"text": "Describe the document?", "files": ["example_images/document.jpg"]}],
     [{"text": "What does this say?", "files": ["example_images/math.jpg"]}],
     [{"text": "What is this UI about?", "files": ["example_images/s2w_example.png"]}],
     [{"text": "Where do the severe droughts happen according to this diagram?", "files": ["example_images/examples_weather_events.png"]}],
-
+    [{"text": "@video-infer Explain the content of the video.", "files": ["example_videos/sample_video.mp4"]}],
 ]
 
 demo = gr.ChatInterface(
     fn=model_inference,
     description="# **Qwen2.5-VL-7B-Instruct**",
     examples=examples,
-    textbox=gr.MultimodalTextbox(label="Query Input", file_types=["image"], file_count="multiple"),
+    textbox=gr.MultimodalTextbox(label="Query Input", file_types=["image", "video"], file_count="multiple"),
     stop_btn="Stop Generation",
     multimodal=True,
     cache_examples=False,