redesign the app in two-step questions: 1. show images, 2. ask the question

app.py

@@ -7,7 +7,7 @@ import time
 from pathlib import Path
 from huggingface_hub import CommitScheduler, HfApi
 
-from src.utils import load_words, load_image_and_saliency, load_example_images
+from src.utils import load_words, load_image_and_saliency, load_example_images, load_csv_concepts
 from src.style import css
 from src.user import UserID
 
@@ -31,29 +31,9 @@ def main():
         #user_id = gr.State(load_global_variable())
         answers = gr.State([])
 
-        with gr.Row():
-            target_img_label = gr.Markdown(f"Target image: **{class_names[user_state.value]}**")
-            gr.Markdown("Grad-cam")
-            gr.Markdown("Lime")
-            gr.Markdown("Sidu")
-            gr.Markdown("Rise")
+        concepts = load_csv_concepts(data_dir)
 
-        with gr.Row():
-            count = user_state if isinstance(user_state, int) else user_state.value
-            images = load_image_and_saliency(count, data_dir)
-            target_img = gr.Image(images[0], elem_classes="main-image")
-            saliency_gradcam = gr.Image(images[1], elem_classes="main-image")
-            saliency_lime = gr.Image(images[2], elem_classes="main-image")
-            saliency_sidu = gr.Image(images[3], elem_classes="main-image")
-            saliency_rise = gr.Image(images[4], elem_classes="main-image")
-            
-        with gr.Row():
-            dropdown1 = gr.Dropdown(choices=options, label="grad-cam")
-            dropdown2 = gr.Dropdown(choices=options, label="lime")
-            dropdown3 = gr.Dropdown(choices=options, label="sidu")
-            dropdown4 = gr.Dropdown(choices=options, label="rise")
-
-        gr.Markdown("### Image examples of the same class")
+        gr.Markdown("### Image examples")
         with gr.Row():
             count = user_state if isinstance(user_state, int) else user_state.value
             images = load_example_images(count, data_dir)
@@ -73,42 +53,75 @@ def main():
             img14 = gr.Image(images[13])
             img15 = gr.Image(images[14])
             img16 = gr.Image(images[15])
+
+        with gr.Row():
+            target_img_label = gr.Markdown(f"Target image: **{class_names[user_state.value]}**")
+            gr.Markdown("Grad-cam")
+            gr.Markdown("Lime")
+            gr.Markdown("Sidu")
+            gr.Markdown("Rise")
+
+        with gr.Row():
+            count = user_state if isinstance(user_state, int) else user_state.value
+            images = load_image_and_saliency(count, data_dir)
+            target_img = gr.Image(images[0], elem_classes="main-image delay", visible=False)
+            saliency_gradcam = gr.Image(images[1], elem_classes="main-image", visible=False)
+            saliency_lime = gr.Image(images[2], elem_classes="main-image", visible=False)
+            saliency_sidu = gr.Image(images[4], elem_classes="main-image", visible=False)
+            saliency_rise = gr.Image(images[3], elem_classes="main-image", visible=False)
             
-        submit_button = gr.Button("Submit")
+        count = user_state if isinstance(user_state, int) else user_state.value
+        row = concepts.iloc[count]        
+        question = gr.Markdown(f"### Sort the following saliency maps according to which of them better highlights the following concepts: {row[1]}, {row[2]} , {row[3]}?", visible=False)
+        with gr.Row():
+            dropdown1 = gr.Dropdown(choices=options, label="grad-cam", visible=False)
+            dropdown2 = gr.Dropdown(choices=options, label="lime", visible=False)
+            dropdown3 = gr.Dropdown(choices=options, label="sidu", visible=False)
+            dropdown4 = gr.Dropdown(choices=options, label="rise", visible=False)
+        
+        continue_button = gr.Button("Continue")
+        submit_button = gr.Button("Submit", visible=False)
         finish_button = gr.Button("Finish", visible=False)
 
-        def update_images(dropdown1, dropdown2, dropdown3, dropdown4, user_state):
-
+        def update_images(user_state):
             count = user_state if isinstance(user_state, int) else user_state.value
             if count < config['dataset'][config['dataset']['name']]['n_classes']:
                 images = load_image_and_saliency(count, data_dir)
-                target_img = gr.Image(images[0], elem_classes="main-image")
-                saliency_gradcam = gr.Image(images[1], elem_classes="main-image")
-                saliency_lime = gr.Image(images[2], elem_classes="main-image")
-                saliency_sidu = gr.Image(images[3], elem_classes="main-image")
-                saliency_rise = gr.Image(images[4], elem_classes="main-image")
-
+                
                 # image examples
                 images = load_example_images(count, data_dir)
-                img1 = gr.Image(images[0])
-                img2 = gr.Image(images[1])
-                img3 = gr.Image(images[2])
-                img4 = gr.Image(images[3])
-                img5 = gr.Image(images[4])
-                img6 = gr.Image(images[5])
-                img7 = gr.Image(images[6])
-                img8 = gr.Image(images[7])
-                img9 = gr.Image(images[8])
-                img10 = gr.Image(images[9])
-                img11 = gr.Image(images[10])
-                img12 = gr.Image(images[11])
-                img13 = gr.Image(images[12])
-                img14 = gr.Image(images[13])
-                img15 = gr.Image(images[14])
-                img16 = gr.Image(images[15])
-                return target_img, saliency_gradcam, saliency_lime, saliency_rise, saliency_sidu, img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, img11, img12, img13, img14, img15, img16
+                img1 = gr.Image(images[0], visible=True)
+                img2 = gr.Image(images[1], visible=True)
+                img3 = gr.Image(images[2], visible=True)
+                img4 = gr.Image(images[3], visible=True)
+                img5 = gr.Image(images[4], visible=True)
+                img6 = gr.Image(images[5], visible=True)
+                img7 = gr.Image(images[6], visible=True)
+                img8 = gr.Image(images[7], visible=True)
+                img9 = gr.Image(images[8], visible=True)
+                img10 = gr.Image(images[9], visible=True)
+                img11 = gr.Image(images[10], visible=True)
+                img12 = gr.Image(images[11], visible=True)
+                img13 = gr.Image(images[12], visible=True)
+                img14 = gr.Image(images[13], visible=True)
+                img15 = gr.Image(images[14], visible=True)
+                img16 = gr.Image(images[15], visible=True)
+                return img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, img11, img12, img13, img14, img15, img16
             else:
-                return target_img, saliency_gradcam, saliency_lime, saliency_rise, saliency_sidu, img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, img11, img12, img13, img14, img15, img16
+                return img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, img11, img12, img13, img14, img15, img16
+
+        def update_saliencies(dropdown1, dropdown2, dropdown3, dropdown4, user_state):
+            count = user_state if isinstance(user_state, int) else user_state.value
+            if count < config['dataset'][config['dataset']['name']]['n_classes']:
+                images = load_image_and_saliency(count, data_dir)
+                target_img = gr.Image(images[0], elem_classes="main-image", visible=True)
+                saliency_gradcam = gr.Image(images[1], elem_classes="main-image", visible=True)
+                saliency_lime = gr.Image(images[2], elem_classes="main-image", visible=True)
+                saliency_sidu = gr.Image(images[4], elem_classes="main-image", visible=True)
+                saliency_rise = gr.Image(images[3], elem_classes="main-image", visible=True)
+                return target_img, saliency_gradcam, saliency_lime, saliency_rise, saliency_sidu
+            else:
+                return target_img, saliency_gradcam, saliency_lime, saliency_rise, saliency_sidu
 
         def update_state(state):
             count = state if isinstance(state, int) else state.value
@@ -116,22 +129,49 @@ def main():
 
         def update_img_label(state):
             count = state if isinstance(state, int) else state.value
-            return f"### Target image: {class_names[count]}"
+            return f" Target image: **{class_names[count]}**"
+
+        def update_buttons():
+            submit_button = gr.Button("Submit", visible=False)
+            continue_button = gr.Button("Continue", visible=True)
+            return continue_button, submit_button
 
-        def update_buttons(state):
+        def show_view(state):
             count = state if isinstance(state, int) else state.value
             max_images = config['dataset'][config['dataset']['name']]['n_classes']
             finish_button = gr.Button("Finish", visible=(count == max_images-1))
             submit_button = gr.Button("Submit", visible=(count != max_images-1))
-            return submit_button, finish_button
+            continue_button = gr.Button("Continue", visible=False)
+            return continue_button, submit_button, finish_button
+
+
+        def hide_view():
+            target_img = gr.Image(images[0], elem_classes="main-image", visible=False)
+            saliency_gradcam = gr.Image(images[1], elem_classes="main-image", visible=False)
+            saliency_lime = gr.Image(images[2], elem_classes="main-image", visible=False)
+            saliency_sidu = gr.Image(images[4], elem_classes="main-image", visible=False)
+            saliency_rise = gr.Image(images[3], elem_classes="main-image", visible=False)
+            question = gr.Markdown(f"### Sort the following saliency maps according to which of them better highlights the following concepts: {row[1]}, {row[2]} , {row[3]}?", visible=False)
+            dropdown1 = gr.Dropdown(choices=options, label="grad-cam", visible=False)
+            dropdown2 = gr.Dropdown(choices=options, label="lime", visible=False)
+            dropdown3 = gr.Dropdown(choices=options, label="sidu", visible=False)
+            dropdown4 = gr.Dropdown(choices=options, label="rise", visible=False)
+            return target_img, saliency_gradcam, saliency_lime, saliency_sidu, saliency_rise, question, dropdown1, dropdown2, dropdown3, dropdown4
+
 
         def update_dropdowns():
-            dp1 = gr.Dropdown(choices=options, value=options[0], label="grad-cam")
-            dp2 = gr.Dropdown(choices=options, value=options[0], label="lime")
-            dp3 = gr.Dropdown(choices=options, value=options[0], label="sidu")
-            dp4 = gr.Dropdown(choices=options, value=options[0], label="rise")
+            dp1 = gr.Dropdown(choices=options, value=options[0], label="grad-cam", visible=True)
+            dp2 = gr.Dropdown(choices=options, value=options[0], label="lime", visible=True)
+            dp3 = gr.Dropdown(choices=options, value=options[0], label="sidu", visible=True)
+            dp4 = gr.Dropdown(choices=options, value=options[0], label="rise", visible=True)
             return dp1, dp2, dp3, dp4
 
+        def update_questions(state):
+            concepts = load_csv_concepts(data_dir)
+            count = state if isinstance(state, int) else state.value
+            row = concepts.iloc[count]
+            return gr.Markdown(f"### Sort the following saliency maps according to which of them better highlights the following concepts: {row[1]}, {row[2]} , {row[3]}?", visible=True)
+
         def redirect():
             pass
 
@@ -145,7 +185,7 @@ def main():
             JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
             JSON_DATASET_PATH = JSON_DATASET_DIR / f"train-{uuid4()}.json"
             scheduler = CommitScheduler(
-                repo_id="example-space-to-dataset-json",
+                repo_id=f"results_{config['dataset']['name']}_{config['results']['exp1_dir']}",  # The repo id
                 repo_type="dataset",
                 folder_path=JSON_DATASET_DIR,
                 path_in_repo="data",
@@ -196,19 +236,40 @@ def main():
             update_img_label,
             inputs=user_state,
             outputs=target_img_label
+        ).then(
+            update_images,
+            inputs=user_state,
+            outputs=[img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, img11, img12, img13, img14, img15, img16]    
         ).then(
             update_buttons,
+            outputs={continue_button, submit_button}
+        ).then(
+            hide_view,
+            outputs={target_img, saliency_gradcam, saliency_lime, saliency_sidu, saliency_rise, question, dropdown1, dropdown2, dropdown3, dropdown4}
+        )
+       
+        continue_button.click(
+            show_view,
+            inputs=user_state,
+            outputs={continue_button, submit_button, finish_button}
+        ).then(
+            update_img_label,
             inputs=user_state,
-            outputs={submit_button, finish_button}
+            outputs=target_img_label
         ).then(
-            update_images, 
+            update_saliencies, 
             inputs=[dropdown1, dropdown2, dropdown3, dropdown4, user_state], 
-            outputs={target_img, saliency_gradcam, saliency_lime, saliency_sidu, saliency_rise, img1, img2, img3, img4, img5, img6, img7, img8, img9, img10, img11, img12, img13, img14, img15, img16},
+            outputs={target_img, saliency_gradcam, saliency_lime, saliency_sidu, saliency_rise},
+        ).then(
+            update_questions,
+            inputs=user_state,
+            outputs=question
         ).then(
             update_dropdowns,
             outputs={dropdown1, dropdown2, dropdown3, dropdown4}
         )
-       
+
+
         finish_button.click(
             add_answer, inputs=[dropdown1, dropdown2, dropdown3, dropdown4, answers],outputs=answers
         ).then(

data/intel_image/concepts_by_class.csv

@@ -0,0 +1,7 @@
+class, concept1, concept2, concept3, concept4, concept5, concept6, concept7, concept8, concept9, concept10, concept11, concept12, concept13, concept14, concept15, concept16
+buildings, Roof, Window, Facade, Wall, Boat, Tree, Sky, Car, Streetlights, Sidewalk, Beach, Vegetation, Water, Mountain Peak, Rock, Ice
+forest, Vegetation, Tree, Water, Sidewalk, Facade, Sky, Beach, Wall, Rock, Window, Ice, Roof, Streetlights, Car, Mountain Peak, Boat
+glacier, Ice, Rock, Mountain Peak, Water, Wall, Beach, Sky, Vegetation, Sidewalk, Facade, Roof, Tree, Window, Boat, Streetlights, Car
+mountain, Mountain Peak, Rock, Vegetation, Sky, Tree, Ice, Water, Beach, Wall, Facade, Roof, Boat, Sidewalk, Window, Streetlights, Car
+sea, Water, Boat, Beach, Sky, Rock, Sidewalk, Wall, Ice, Roof, Vegetation, Facade, Mountain Peak, Tree, Streetlights, Window, Car
+street, Car, Streetlights, Sidewalk, Boat, Wall, Facade, Tree, Roof, Beach, Sky, Window, Vegetation, Water, Rock, Mountain Peak, Ice

src/utils.py

@@ -33,7 +33,10 @@ def load_words(idx):
     return words
 
 
-
+def load_csv_concepts(data_dir):
+    # Load data from csv
+    data = pd.read_csv(os.path.join(data_dir, 'concepts_by_class.csv'))
+    return data