adding closest samples

app.py

@@ -9,6 +9,7 @@ if os.getenv('SYSTEM') == 'spaces':
     subprocess.call('pip install git+https://github.com/cocodataset/panopticapi.git'.split())
     subprocess.call('pip install python-dotenv'.split())
     subprocess.call('pip install torch torchvision '.split())
+    subprocess.call('pip install xplique'.split())
 
 import gradio as gr
 from huggingface_hub import snapshot_download
@@ -19,13 +20,38 @@ import numpy as np
 import gradio as gr
 import glob
 from inference_sam import segmentation_sam
-
+from explanations import explain
+from inference_resnet import get_triplet_model
 import pathlib
+import tensorflow as tf
+from closest_sample import get_images
 
 if not os.path.exists('images'):
     REPO_ID='Serrelab/image_examples_gradio'
     snapshot_download(repo_id=REPO_ID, token=os.environ.get('READ_TOKEN'),repo_type='dataset',local_dir='images')
 
+def get_model(model_name):
+   
+        
+    if model_name=='Mummified 170':
+        n_classes = 170
+        model = get_triplet_model(input_shape = (600, 600, 3),
+                        embedding_units = 256,
+                        embedding_depth = 2,
+                        backbone_class=tf.keras.applications.ResNet50V2,
+                        nb_classes = n_classes,load_weights=False,finer_model=True,backbone_name ='Resnet50v2')
+        model.load_weights('model_classification/mummified-170.h5')
+    elif model_name=='Rock 170':
+        n_classes = 171
+        model = get_triplet_model(input_shape = (600, 600, 3),
+                        embedding_units = 256,
+                        embedding_depth = 2,
+                        backbone_class=tf.keras.applications.ResNet50V2,
+                        nb_classes = n_classes,load_weights=False,finer_model=True,backbone_name ='Resnet50v2')
+        model.load_weights('model_classification/rock-170.h5')
+    else:
+        return 'Error' 
+    return model,n_classes
 
 def segment_image(input_image):
     img = segmentation_sam(input_image)
@@ -34,24 +60,54 @@ def segment_image(input_image):
 def classify_image(input_image, model_name):
     if 'Rock 170' ==model_name:
         from inference_resnet import inference_resnet_finer
-        result = inference_resnet_finer(input_image,model_name,n_classes=171)
+        model,n_classes= get_model(model_name)
+        result = inference_resnet_finer(input_image,model,size=600,n_classes=n_classes)
         return result 
     elif 'Mummified 170' ==model_name:
         from inference_resnet import inference_resnet_finer
-        result = inference_resnet_finer(input_image,model_name,n_classes=170)
+        model, n_classes= get_model(model_name)
+        result = inference_resnet_finer(input_image,model,size=600,n_classes=n_classes)
         return result 
     if 'Fossils 19' ==model_name:
         from inference_beit import inference_dino
+        model,n_classes = get_model(model_name)
         return inference_dino(input_image,model_name)
     return None
 
-def find_closest(input_image):
-    return None 
+def get_embeddings(input_image,model_name):
+    if 'Rock 170' ==model_name:
+        from inference_resnet import inference_resnet_embedding
+        model,n_classes= get_model(model_name)
+        result = inference_resnet_embedding(input_image,model,size=600,n_classes=n_classes)
+        return result 
+    elif 'Mummified 170' ==model_name:
+        from inference_resnet import inference_resnet_embedding
+        model, n_classes= get_model(model_name)
+        result = inference_resnet_embedding(input_image,model,size=600,n_classes=n_classes)
+        return result 
+    if 'Fossils 19' ==model_name:
+        from inference_beit import inference_dino
+        model,n_classes = get_model(model_name)
+        return inference_dino(input_image,model_name)
+    return None
+    
 
+def find_closest(input_image,model_name):
+    embedding = get_embeddings(input_image,model_name)
+    paths = get_images(embedding)
+    return paths
+
+def explain_image(input_image,model_name):
+    model,n_classes= get_model(model_name)
+    saliency, integrated, smoothgrad = explain(model,input_image,n_classes=n_classes)
+    #original =  saliency + integrated + smoothgrad 
+    print('done')
+    return saliency, integrated, smoothgrad,
+    
 
 with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
     
-    with gr.Tab(" 19 Classes Support"):
+    with gr.Tab(" Florrissant Fossils"):
     
         with gr.Row():
             with gr.Column():
@@ -64,10 +120,10 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
                 #classify_segmented_button = gr.Button("Classify Segmented Image")
                 
             with gr.Column():
-                drop_2 = gr.Dropdown(
-                    ["Mummified 170", "Rock 170", "Fossils 19"],
+                model_name = gr.Dropdown(
+                    ["Mummified 170", "Rock 170"],
                     multiselect=False,
-                    value=["Rock 170"],
+                    value="Rock 170",
                     label="Model",
                     interactive=True,
                 )
@@ -81,24 +137,24 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
             samples=[[path.as_posix()] for path in paths  if 'leaves' in  str(path) ][:19]
             examples_leaves = gr.Examples(samples, inputs=input_image,examples_per_page=5,label='Leaves Examples from the dataset')
             
-        with gr.Accordion("Using Diffuser"):
-            with gr.Column(): 
-                prompt = gr.Textbox(lines=1, label="Prompt")
-                output_image = gr.Image(label="Output")
-                generate_button = gr.Button("Generate Leave")    
-            with gr.Column():
-                class_predicted2 = gr.Label(label='Class Predicted from diffuser')
-                classify_button = gr.Button("Classify Image")
+        # with gr.Accordion("Using Diffuser"):
+        #     with gr.Column(): 
+        #         prompt = gr.Textbox(lines=1, label="Prompt")
+        #         output_image = gr.Image(label="Output")
+        #         generate_button = gr.Button("Generate Leave")    
+        #     with gr.Column():
+        #         class_predicted2 = gr.Label(label='Class Predicted from diffuser')
+        #         classify_button = gr.Button("Classify Image")
             
         
         with gr.Accordion("Explanations "):
             gr.Markdown("Computing Explanations from the model")
             with gr.Row():
-                original_input = gr.Image(label="Original Frame")
+                #original_input = gr.Image(label="Original Frame")
                 saliency  = gr.Image(label="saliency")
-                gradcam = gr.Image(label='gradcam')
-                guided_gradcam = gr.Image(label='guided gradcam')
-                guided_backprop = gr.Image(label='guided backprop')
+                gradcam = gr.Image(label='integraged gradients')
+                guided_gradcam = gr.Image(label='gradcam')
+                #guided_backprop = gr.Image(label='guided backprop')
             generate_explanations = gr.Button("Generate Explanations")
                 
         with gr.Accordion('Closest Images'):
@@ -112,8 +168,10 @@ with gr.Blocks(theme='sudeepshouche/minimalist') as demo:
             find_closest_btn = gr.Button("Find Closest Images")
             
         segment_button.click(segment_image, inputs=input_image, outputs=segmented_image)
-        classify_image_button.click(classify_image, inputs=[input_image,drop_2], outputs=class_predicted)
-        #classify_segmented_button.click(classify_image, inputs=[segmented_image,drop_2], outputs=class_predicted)
+        classify_image_button.click(classify_image, inputs=[input_image,model_name], outputs=class_predicted)
+        generate_explanations.click(explain_image, inputs=[input_image,model_name], outputs=[saliency,gradcam,guided_gradcam])
+        find_closest_btn.click(find_closest, inputs=[input_image,model_name], outputs=[closest_image_0,closest_image_1,closest_image_2,closest_image_3,closest_image_4])
+        #classify_segmented_button.click(classify_image, inputs=[segmented_image,model_name], outputs=class_predicted)
         
 demo.queue()     
    

inference_resnet.py

@@ -12,9 +12,10 @@ from huggingface_hub import snapshot_download
 from labels import lookup_170
 import numpy as np 
 
+if not os.path.exists('model_classification'):
 
-REPO_ID='Serrelab/fossil_classification_models'
-snapshot_download(repo_id=REPO_ID, token=os.environ.get('READ_TOKEN'),repo_type='model',local_dir='model_classification')
+    REPO_ID='Serrelab/fossil_classification_models'
+    snapshot_download(repo_id=REPO_ID, token=os.environ.get('READ_TOKEN'),repo_type='model',local_dir='model_classification')
 
 
 def get_model(base_arch='Nasnet',weights='imagenet',input_shape=(600,600,3),classes=64500):
@@ -146,19 +147,19 @@ def parse_results(top_n,logits):
         results[label] = float(logits[n])
     return results 
 
-def inference_resnet_finer(x,type_model,size=576,n_classes=170,n_top=10):
+def inference_resnet_embedding(x,model,size=576,n_classes=170,n_top=10):
+        
+        
+    cropped = _clever_crop(x,(size,size))[0]
+    prep = preprocess(cropped,size=size)
+    embedding = model.predict(np.array([prep]))[0][0]
+   
+    
+    return embedding 
+
+def inference_resnet_finer(x,model,size=576,n_classes=170,n_top=10):
+    
     
-    model = get_triplet_model(input_shape = (size, size, 3),
-                        embedding_units = 256,
-                        embedding_depth = 2,
-                        backbone_class=tf.keras.applications.ResNet50V2,
-                        nb_classes = n_classes,load_weights=False,finer_model=True,backbone_name ='Resnet50v2')
-    if type_model=='Mummified 170':
-        model.load_weights('model_classification/mummified-170.h5')
-    elif type_model=='Rock 170':
-        model.load_weights('model_classification/rock-170.h5')
-    else:
-        return 'Error' 
     cropped = _clever_crop(x,(size,size))[0]
     prep = preprocess(cropped,size=size)
     logits = tf.nn.softmax(model.predict(np.array([prep]))[1][0]).cpu().numpy()

inference_sam.py

@@ -12,10 +12,11 @@ from math import ceil
 import os
 from huggingface_hub import snapshot_download
 
-REPO_ID='Serrelab/SAM_Leaves'
-snapshot_download(repo_id=REPO_ID, token=os.environ.get('READ_TOKEN'),repo_type='model',local_dir='model')
+if not os.path.exists('model'):
+  REPO_ID='Serrelab/SAM_Leaves'
+  snapshot_download(repo_id=REPO_ID, token=os.environ.get('READ_TOKEN'),repo_type='model',local_dir='model')
 
-sam = sam_model_registry["default"]("model/sam_02-06_dice_mse_0.pth")
+sam = sam_model_registry["default"]("/home/irodri15/Documents/Projects/Fossils/fossil_app/model/sam_02-06_dice_mse_0.pth")
 sam.cuda()
 predictor = SamPredictor(sam)
 
@@ -172,4 +173,4 @@ def segmentation_sam(x,SIZE=384):
     data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
     data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
     plt.close()
-    return data 
+    return data 

pre-requirements.txt

@@ -3,4 +3,5 @@ opencv-python-headless==4.5.5.64
 openmim==0.1.5
 torch==1.11.0
 torchvision==0.12.0
-tensorflow==2.8 
+tensorflow==2.8 
+xplique