add tabs

.python-version

@@ -0,0 +1 @@
+3.10

app.py

@@ -1,9 +1,13 @@
 import os
 import logging
 import sys
+from config import WEAVE_PROJECT, WANDB_API_KEY
+import weave
+
+weave.init(WEAVE_PROJECT)
 
 # Function to get logging level from environment variable
-def get_logging_level(default_level=logging.DEBUG):  # Default to DEBUG for detailed logs
+def get_logging_level(default_level=logging.INFO):  # Default to DEBUG for detailed logs
     log_level_str = os.getenv('VISION_AGENT_LOG_LEVEL', '').upper()
     if log_level_str == 'DEBUG':
         return logging.DEBUG
@@ -18,24 +22,9 @@ def get_logging_level(default_level=logging.DEBUG):  # Default to DEBUG for deta
     else:
         return default_level
 
-# Set up logging before importing the library
-logging_level = get_logging_level()
-logging.basicConfig(stream=sys.stdout, level=logging_level)
-_LOGGER = logging.getLogger(__name__)
-
-# Explicitly set logging level for the vision-agent library
-vision_agent_logger = logging.getLogger('vision_agent')
-vision_agent_logger.setLevel(logging_level)
-
-# Set logging level for Hugging Face libraries
-hf_hub_logger = logging.getLogger('huggingface_hub')
-hf_hub_logger.setLevel(logging_level)
-
-datasets_logger = logging.getLogger('datasets')
-datasets_logger.setLevel(logging_level)
-
-# Print the logging level to verify it's set correctly
-print(f"Logging level set to: {logging.getLevelName(logging_level)}")
+# Initialize logger
+logging.basicConfig(level=get_logging_level(), format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger('vision_agent')
 
 from huggingface_hub import login
 import time
@@ -44,7 +33,7 @@ from typing import *
 from pillow_heif import register_heif_opener
 register_heif_opener()
 import vision_agent as va
-from vision_agent.tools import register_tool, load_image, owl_v2, overlay_bounding_boxes, save_image
+from vision_agent.tools import register_tool, load_image, owl_v2, grounding_dino, florencev2_object_detection, overlay_bounding_boxes, save_image
 
 # Perform login using the token
 hf_token = os.getenv("HF_TOKEN")
@@ -53,7 +42,8 @@ login(token=hf_token, add_to_git_credential=True)
 import numpy as np
 from PIL import Image
 
-def detect_brain_tumor(image, seg_input, debug: bool = True):
+@weave.op()
+def detect_brain_tumor_owlv2(image, seg_input, debug: bool = True):
     """
     Detects a brain tumor in the given image and returns the annotated image.
 
@@ -66,18 +56,66 @@ def detect_brain_tumor(image, seg_input, debug: bool = True):
         tuple: (numpy array of image, list of (label, (x1, y1, x2, y2)) tuples)
     """
     if debug:
-        _LOGGER.debug(f"Image received, shape: {image.shape}")
+        logger.debug(f"Image received, shape: {image.shape}")
 
     # Step 2: Detect brain tumor using owl_v2
     prompt = "detect brain tumor"
     detections = owl_v2(prompt, image)
     if debug:
-        _LOGGER.debug(f"Raw detections: {detections}")
+        logger.debug(f"Raw detections: {detections}")
+
+    # Step 3: Overlay bounding boxes on the image
+    image_with_bboxes = overlay_bounding_boxes(image, detections)
+    if debug:
+        logger.debug("Bounding boxes overlaid on the image")
+
+    # Prepare annotations for AnnotatedImage output
+    annotations = []
+    for detection in detections:
+        label = detection['label']
+        score = detection['score']
+        bbox = detection['bbox']
+        x1, y1, x2, y2 = bbox
+        # Convert normalized coordinates to pixel coordinates
+        height, width = image.shape[:2]
+        x1, y1, x2, y2 = int(x1*width), int(y1*height), int(x2*width), int(y2*height)
+        annotations.append(((x1, y1, x2, y2), f"{label} {score:.2f}"))
+
+    if debug:
+        logger.debug(f"Annotations: {annotations}")
+
+    # Convert image to numpy array if it's not already
+    if isinstance(image_with_bboxes, Image.Image):
+        image_with_bboxes = np.array(image_with_bboxes)
+
+    return (image_with_bboxes, annotations)
+
+@weave.op()
+def detect_brain_tumor_dino(image, seg_input, debug: bool = True):
+    """
+    Detects a brain tumor in the given image and returns the annotated image.
+
+    Parameters:
+        image: The input image (as numpy array provided by Gradio).
+        seg_input: The segmentation input (not used in this function, but required for Gradio).
+        debug (bool): Flag to enable logging for debugging purposes.
+
+    Returns:
+        tuple: (numpy array of image, list of (label, (x1, y1, x2, y2)) tuples)
+    """
+    if debug:
+        logger.debug(f"Image received, shape: {image.shape}")
+
+    # Step 2: Detect brain tumor using grounding_dino
+    prompt = "detect brain tumor"
+    detections = grounding_dino(prompt, image)
+    if debug:
+        logger.debug(f"Raw detections: {detections}")
 
     # Step 3: Overlay bounding boxes on the image
     image_with_bboxes = overlay_bounding_boxes(image, detections)
     if debug:
-        _LOGGER.debug("Bounding boxes overlaid on the image")
+        logger.debug("Bounding boxes overlaid on the image")
 
     # Prepare annotations for AnnotatedImage output
     annotations = []
@@ -92,7 +130,55 @@ def detect_brain_tumor(image, seg_input, debug: bool = True):
         annotations.append(((x1, y1, x2, y2), f"{label} {score:.2f}"))
 
     if debug:
-        _LOGGER.debug(f"Annotations: {annotations}")
+        logger.debug(f"Annotations: {annotations}")
+
+    # Convert image to numpy array if it's not already
+    if isinstance(image_with_bboxes, Image.Image):
+        image_with_bboxes = np.array(image_with_bboxes)
+
+    return (image_with_bboxes, annotations)
+
+@weave.op()
+def detect_brain_tumor_florence2(image, seg_input, debug: bool = True):
+    """
+    Detects a brain tumor in the given image and returns the annotated image.
+
+    Parameters:
+        image: The input image (as numpy array provided by Gradio).
+        seg_input: The segmentation input (not used in this function, but required for Gradio).
+        debug (bool): Flag to enable logging for debugging purposes.
+
+    Returns:
+        tuple: (numpy array of image, list of (label, (x1, y1, x2, y2)) tuples)
+    """
+    if debug:
+        logger.debug(f"Image received, shape: {image.shape}")
+
+    # Step 2: Detect brain tumor using florencev2 - NO PROMPT
+    prompt = "detect brain tumor"
+    detections = florencev2_object_detection(prompt)
+    if debug:
+        logger.debug(f"Raw detections: {detections}")
+
+    # Step 3: Overlay bounding boxes on the image
+    image_with_bboxes = overlay_bounding_boxes(image, detections)
+    if debug:
+        logger.debug("Bounding boxes overlaid on the image")
+
+    # Prepare annotations for AnnotatedImage output
+    annotations = []
+    for detection in detections:
+        label = detection['label']
+        score = detection['score']
+        bbox = detection['bbox']
+        x1, y1, x2, y2 = bbox
+        # Convert normalized coordinates to pixel coordinates
+        height, width = image.shape[:2]
+        x1, y1, x2, y2 = int(x1*width), int(y1*height), int(x2*width), int(y2*height)
+        annotations.append(((x1, y1, x2, y2), f"{label} {score:.2f}"))
+
+    if debug:
+        logger.debug(f"Annotations: {annotations}")
 
     # Convert image to numpy array if it's not already
     if isinstance(image_with_bboxes, Image.Image):
@@ -104,7 +190,7 @@ INTRO_TEXT="# 🔬🧠 OmniScience -- Agentic Imaging Analysis 🤖🧫"
 
 with gr.Blocks(css="style.css") as demo:
     gr.Markdown(INTRO_TEXT)
-    with gr.Tab("Segment/Detect"):
+    with gr.Tab("Object Detection - Owl V2"):
         with gr.Row():
             with gr.Column():
                 image = gr.Image(type="numpy")
@@ -135,10 +221,82 @@ with gr.Blocks(css="style.css") as demo:
             annotated_image
         ]
         seg_btn.click(
-            fn=detect_brain_tumor,
+            fn=detect_brain_tumor_owlv2,
             inputs=seg_inputs,
             outputs=seg_outputs,
         )
 
+    with gr.Tab("Object Detection - DINO"):
+        with gr.Row():
+            with gr.Column():
+                image = gr.Image(type="numpy")
+                seg_input = gr.Text(label="Entities to Segment/Detect", value="detect brain tumor")
+        
+            with gr.Column():
+                annotated_image = gr.AnnotatedImage(label="Output")
+
+        seg_btn = gr.Button("Submit")    
+        examples = [["./examples/194_jpg.rf.3e3dd592d034bb5ee27a978553819f42.jpg", "detect brain tumor"],
+                    ["./examples/239_jpg.rf.3dcc0799277fb78a2ab21db7761ccaeb.jpg", "detect brain tumor"],
+                    ["./examples/1385_jpg.rf.3c67cb92e2922dba0e6dba86f69df40b.jpg", "detect brain tumor"],
+                    ["./examples/1491_jpg.rf.3c658e83538de0fa5a3f4e13d7d85f12.jpg", "detect brain tumor"],
+                    ["./examples/1550_jpg.rf.3d067be9580ec32dbee5a89c675d8459.jpg", "detect brain tumor"],
+                    ["./examples/2256_jpg.rf.3afd7903eaf3f3c5aa8da4bbb928bc19.jpg", "detect brain tumor"],
+                    ["./examples/2871_jpg.rf.3b6eadfbb369abc2b3bcb52b406b74f2.jpg", "detect brain tumor"],
+                    ["./examples/2921_jpg.rf.3b952f91f27a6248091e7601c22323ad.jpg", "detect brain tumor"],
+                    ]
+        gr.Examples(
+            examples=examples,
+            inputs=[image, seg_input],
+        )
+        seg_inputs = [
+            image,
+            seg_input
+            ]
+        seg_outputs = [
+            annotated_image
+        ]
+        seg_btn.click(
+            fn=detect_brain_tumor_dino,
+            inputs=seg_inputs,
+            outputs=seg_outputs,
+        )
+
+    with gr.Tab("Object Detection - Florence2"):
+        with gr.Row():
+            with gr.Column():
+                image = gr.Image(type="numpy")
+                seg_input = gr.Text(label="Entities to Segment/Detect - PROMPT IS NOT PASSED TO FLORENCE2", value="detect brain tumor")
+        
+            with gr.Column():
+                annotated_image = gr.AnnotatedImage(label="Output")
+
+        seg_btn = gr.Button("Submit")    
+        examples = [["./examples/194_jpg.rf.3e3dd592d034bb5ee27a978553819f42.jpg", "detect brain tumor"],
+                    ["./examples/239_jpg.rf.3dcc0799277fb78a2ab21db7761ccaeb.jpg", "detect brain tumor"],
+                    ["./examples/1385_jpg.rf.3c67cb92e2922dba0e6dba86f69df40b.jpg", "detect brain tumor"],
+                    ["./examples/1491_jpg.rf.3c658e83538de0fa5a3f4e13d7d85f12.jpg", "detect brain tumor"],
+                    ["./examples/1550_jpg.rf.3d067be9580ec32dbee5a89c675d8459.jpg", "detect brain tumor"],
+                    ["./examples/2256_jpg.rf.3afd7903eaf3f3c5aa8da4bbb928bc19.jpg", "detect brain tumor"],
+                    ["./examples/2871_jpg.rf.3b6eadfbb369abc2b3bcb52b406b74f2.jpg", "detect brain tumor"],
+                    ["./examples/2921_jpg.rf.3b952f91f27a6248091e7601c22323ad.jpg", "detect brain tumor"],
+                    ]
+        gr.Examples(
+            examples=examples,
+            inputs=[image, seg_input],
+        )
+        seg_inputs = [
+            image,
+            seg_input
+            ]
+        seg_outputs = [
+            annotated_image
+        ]
+        seg_btn.click(
+            fn=detect_brain_tumor_dino,
+            inputs=seg_inputs,
+            outputs=seg_outputs,
+        )        
+
 if __name__ == "__main__":
     demo.queue(max_size=10).launch(debug=True)

config.py

@@ -0,0 +1,8 @@
+# config.py
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+WANDB_API_KEY = os.getenv('WANDB_API_KEY')
+WEAVE_PROJECT = "omniscience-app"

examples/cart1.json

@@ -1,7 +0,0 @@
-{
-  "name": "cart1",
-  "comment": "Computer illustration of CAR-T therapy",
-  "model": "paligemma-3b-mix-224",
-  "prompt": "segment cells",
-  "license": "https://www.cancertherapyadvisor.com/wp-content/uploads/sites/12/2019/09/CAR-T_G_864309570-860x574.jpg"
-}

examples/cart2.json

@@ -1,7 +0,0 @@
-{
-    "name": "cart2",
-    "comment": "Cancerous T-Cells Using Car-T Therapy",
-    "model": "paligemma-3b-mix-224",
-    "prompt": "segment cells",
-    "license": "https://www.coherentmarketinsights.com/blogimg/uploads/2017/02/Cancerous-T-Cells-Using-CAR-T-Therapy.jpg"
-  }

examples/cart3.json

@@ -1,7 +0,0 @@
-{
-    "name": "cart3",
-    "comment": "CAR T cell therapy",
-    "model": "paligemma-3b-mix-224",
-    "prompt": "segment cells",
-    "license": "https://delveinsight-blog.s3.amazonaws.com/blog/wp-content/uploads/2018/12/09022609/adusumillia-car_0_3x2.jpg"
-  }

examples/cnmc1.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc1",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc2.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc2",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc3.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc3",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc4.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc4",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc5.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc5",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc6.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc6",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc7.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc7",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc8.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc8",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/cnmc9.json

@@ -1,7 +0,0 @@
-{
-  "name": "cnmc9",
-  "comment": "cnmc-leukemia-2019",
-  "model": "paligemma-cnmc-ft",
-  "prompt": "Are these cells healthy or cancerous?",
-  "license": "https://www.cancerimagingarchive.net/collection/c-nmc-2019/"
-}

examples/tumor1.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor1",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor10.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor10",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor2.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor2",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor3.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor3",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor4.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor4",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor5.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor5",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor6.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor6",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor7.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor7",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor8.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor8",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

examples/tumor9.json

@@ -1,7 +0,0 @@
-{
-  "name": "tumor9",
-  "comment": "",
-  "model": "",
-  "prompt": "detect cell tumor",
-  "license": ""
-}

requirements.txt

@@ -1,6 +1,7 @@
 git+https://github.com/donbr/vision-agent.git
 git+https://github.com/donbr/vision-agent-tools.git
-spaces
 pillow
 pillow-heif
-weave
+weave
+huggingface-hub
+gradio