fixed multi-layer

.gitignore

@@ -418,4 +418,6 @@ tags
 [._]*.un~
 .vscode
 .github
-generated_samples/
+generated_samples/
+
+results

app.py

@@ -9,6 +9,7 @@ from demo.model_utils import Clip_Utils, Janus_Utils, LLaVA_Utils, ChartGemma_Ut
 import numpy as np
 import matplotlib.pyplot as plt
 import gc
+import os
 import spaces
 from PIL import Image
 
@@ -53,7 +54,7 @@ def multimodal_understanding(model_type,
                              activation_map_method, 
                              visual_pooling_method, 
                              image, question, seed, top_p, temperature, target_token_idx,
-                             visualization_layer_min, visualization_layer_max, focus, response_type):
+                             visualization_layer_min, visualization_layer_max, focus, response_type, chart_type):
     # Clear CUDA cache before generating
     gc.collect()
     if torch.cuda.is_available():
@@ -75,7 +76,8 @@ def multimodal_understanding(model_type,
         if activation_map_method == "GradCAM":
             # Generate Grad-CAM
             all_layers = [layer.layer_norm1 for layer in clip_utils.model.vision_model.encoder.layers]
-            if visualization_layers_min.value != visualization_layers_max.value:
+
+            if visualization_layer_min != visualization_layer_max:
                 target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max-1]
             else:
                 target_layers = [all_layers[visualization_layer_min-1]]
@@ -110,12 +112,6 @@ def multimodal_understanding(model_type,
 
         input_ids = prepare_inputs.input_ids[0].cpu().tolist()
         input_ids_decoded = [tokenizer.decode([input_ids[i]]) for i in range(len(input_ids))]
-        # if model_name.split('-')[0] == "Janus":
-        #     start = 620 
-        # elif model_name.split('-')[0] == "ChartGemma":
-        #     start = 1024
-        # elif model_name.split('-')[0] == "LLaVA": 
-        #     start = 581
 
         if activation_map_method == "GradCAM":
             # target_layers = vl_gpt.vision_model.vision_tower.blocks
@@ -123,11 +119,15 @@ def multimodal_understanding(model_type,
                 all_layers = [block.norm1 for block in vl_gpt.vision_model.vision_tower.blocks]
             else:
                 all_layers = [layer.self_attn for layer in vl_gpt.language_model.model.layers]
-
-            if visualization_layers_min.value != visualization_layers_max.value:
-                target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max-1]
+            
+            print("layer values:", visualization_layer_min, visualization_layer_max)
+            if visualization_layer_min != visualization_layer_max:
+                print("multi layers")
+                target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max]
             else:
+                print("single layer")
                 target_layers = [all_layers[visualization_layer_min-1]]
+            
 
             if model_name.split('-')[0] == "Janus":
                 gradcam = AttentionGuidedCAMJanus(vl_gpt, target_layers)
@@ -165,6 +165,26 @@ def multimodal_understanding(model_type,
 
                         cam.append(cam_i)
 
+    # Collect Results
+    RESULTS_ROOT = "./results"
+    FILES_ROOT = f"{RESULTS_ROOT}/{model_name}/{focus}/{chart_type}/layer{visualization_layer_min}-{visualization_layer_max}"
+    os.makedirs(FILES_ROOT, exist_ok=True)
+    if focus == "Visual Encoder":
+        cam[0].save(f"{FILES_ROOT}/{visual_pooling_method}.png")
+    else:
+        for i, cam_p in enumerate(cam):
+            cam_p.save(f"{FILES_ROOT}/{i}.png")
+            
+    with open(f"{FILES_ROOT}/input_text_decoded.txt", "w") as f:
+        f.write(input_text_decoded)
+        f.close()
+
+    with open(f"{FILES_ROOT}/answer.txt", "w") as f:
+        f.write(answer)
+        f.close()
+            
+
+
     return answer, cam, input_text_decoded
 
 
@@ -235,8 +255,8 @@ def model_slider_change(model_type):
 
         res = (
             gr.Dropdown(choices=["Visualization only", "answer + visualization"], value="answer + visualization", label="response_type"),
-            gr.Slider(minimum=1, maximum=18, value=15, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=18, value=15, step=1, label="visualization layers max"),
+            gr.Slider(minimum=1, maximum=language_model_best_layer, value=language_model_best_layer, step=1, label="visualization layers min"),
+            gr.Slider(minimum=1, maximum=language_model_best_layer, value=language_model_best_layer, step=1, label="visualization layers max"),
             gr.Dropdown(choices=["Language Model"], value="Language Model", label="focus"),
             gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type")
         )
@@ -291,7 +311,7 @@ with gr.Blocks() as demo:
             activation_map_output = gr.Gallery(label="activation Map", height=300, columns=1)
 
         with gr.Column():
-            model_selector = gr.Dropdown(choices=["Clip", "ChartGemma-3B", "Janus-1B", "Janus-7B", "LLaVA-1.5-7B"], value="Clip", label="model")
+            model_selector = gr.Dropdown(choices=["Clip", "ChartGemma-3B", "Janus-Pro-1B", "Janus-Pro-7B", "LLaVA-1.5-7B"], value="Clip", label="model")
             response_type = gr.Dropdown(choices=["Visualization only"], value="Visualization only", label="response_type")
             focus = gr.Dropdown(choices=["Visual Encoder"], value="Visual Encoder", label="focus")
             activation_map_method = gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type")
@@ -339,7 +359,8 @@ with gr.Blocks() as demo:
 
         
 
-    understanding_button = gr.Button("Chat")
+    understanding_button = gr.Button("Submit")
+    chart_type = gr.Textbox(label="Chart Type")
     understanding_output = gr.Textbox(label="Answer")
     understanding_target_token_decoded_output = gr.Textbox(label="Target Token Decoded")
 
@@ -349,67 +370,79 @@ with gr.Blocks() as demo:
         examples=[
 
             [
+                "LineChart",
                 "What was the price of a barrel of oil in February 2020?", 
                 "images/LineChart.png" 
             ],
 
             [
+                "BarChart",
                 "What is the average internet speed in Japan?",
                 "images/BarChart.png"
             ],
 
             [
+                "StackedBar",
                 "What is the cost of peanuts in Seoul?", 
                 "images/StackedBar.png"
             ],
 
-            [ 
+            [
+                "100%StackedBar",
                 "Which country has the lowest proportion of Gold medals?", 
                 "images/Stacked100.png"
             ],
             
             [
+                "PieChart",
                 "What is the approximate global smartphone market share of Samsung?",
                 "images/PieChart.png"
             ],
             
-            [ 
+            [
+                "Histogram",
                 "What distance have customers traveled in the taxi the most?", 
                 "images/Histogram.png"
             ],
 
             [
+                "Scatterplot",
                 "True/False: There is a negative linear relationship between the height and the weight of the 85 males.", 
                 "images/Scatterplot.png"
             ],
 
             [
+                "AreaChart",
                 "What was the average price of pount of coffee beans in October 2019?",
                 "images/AreaChart.png"
             ],
 
             [
+                "StackedArea",
                 "What was the ratio of girls named 'Isla' to girls named 'Amelia' in 2012 in the UK?", 
                 "images/StackedArea.png"
             ],
 
             [
+                "BubbleChart",
                 "Which city's metro system has the largest number of stations?", 
                 "images/BubbleChart.png"
             ],
 
             [ 
+                "Choropleth",
                 "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
                 "images/Choropleth_New.png"
             ],
 
             [
+                "TreeMap",
                 "True/False: eBay is nested in the Software category.", 
                 "images/TreeMap.png"
             ]
             
         ],
-        inputs=[question_input, image_input],
+        inputs=[chart_type, question_input, image_input],
     )
     
 
@@ -418,7 +451,7 @@ with gr.Blocks() as demo:
     understanding_button.click(
         multimodal_understanding,
         inputs=[model_selector, activation_map_method, visual_pooling_method, image_input, question_input, und_seed_input, top_p, temperature, target_token_idx, 
-                visualization_layers_min, visualization_layers_max, focus, response_type],
+                visualization_layers_min, visualization_layers_max, focus, response_type, chart_type],
         outputs=[understanding_output, activation_map_output, understanding_target_token_decoded_output]
     )
     

demo/cam.py

@@ -247,14 +247,10 @@ class AttentionGuidedCAMJanus(AttentionGuidedCAM):
                 
                 act = act.mean(dim=1)
     
-
                 # Compute mean of gradients
                 print("grad_shape:", grad.shape)
                 grad_weights = F.relu(grad.mean(dim=1))
 
-
-                # cam, _ = (act * grad_weights).max(dim=-1)
-                # cam = act * grad_weights
                 cam = act * grad_weights
                 print(cam.shape)
 
@@ -266,17 +262,12 @@ class AttentionGuidedCAMJanus(AttentionGuidedCAM):
 
             # Normalize
             cam_sum = F.relu(cam_sum)
-            # cam_sum = cam_sum - cam_sum.min()
-            # cam_sum = cam_sum / cam_sum.max()
 
             # thresholding
             cam_sum = cam_sum.to(torch.float32)
             percentile = torch.quantile(cam_sum, 0.2)  # Adjust threshold dynamically
             cam_sum[cam_sum < percentile] = 0
 
-            # Reshape
-            # if visual_pooling_method == "CLS":
-            # cam_sum = cam_sum[0, 1:]
 
             # cam_sum shape: [1, seq_len, seq_len]
             cam_sum_lst = []
@@ -300,15 +291,6 @@ class AttentionGuidedCAMJanus(AttentionGuidedCAM):
 
             return cam_sum_lst, grid_size, start
 
-        # Aggregate activations and gradients from ALL layers
-        
-
-
-
-
-
-
-
 
 
 class AttentionGuidedCAMLLaVA(AttentionGuidedCAM):
@@ -376,7 +358,6 @@ class AttentionGuidedCAMLLaVA(AttentionGuidedCAM):
         self.gradients = [layer.get_attn_gradients() for layer in self.target_layers]
         cam_sum = None
 
-        # Ver 2
         for act, grad in zip(self.activations, self.gradients):
 
             print("act shape", act.shape)
@@ -397,13 +378,6 @@ class AttentionGuidedCAMLLaVA(AttentionGuidedCAM):
         cam_sum = F.relu(cam_sum)
         cam_sum = cam_sum.to(torch.float32)
 
-        # thresholding
-        # percentile = torch.quantile(cam_sum, 0.4)  # Adjust threshold dynamically
-        # cam_sum[cam_sum < percentile] = 0
-
-        # Reshape
-        # if visual_pooling_method == "CLS":
-        # cam_sum = cam_sum[0, 1:]
 
         # cam_sum shape: [1, seq_len, seq_len]
         cam_sum_lst = []
@@ -412,7 +386,7 @@ class AttentionGuidedCAMLLaVA(AttentionGuidedCAM):
         for i in range(start_idx, cam_sum_raw.shape[1]):
             cam_sum = cam_sum_raw[0, i, :] # shape: [1: seq_len]
 
-            cam_sum = cam_sum[image_mask].unsqueeze(0) # shape: [1, 1024]
+            cam_sum = cam_sum[image_mask].unsqueeze(0) # shape: [1, img_seq_len]
             print("cam_sum shape: ", cam_sum.shape)
             num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
             grid_size = int(num_patches ** 0.5)
@@ -430,19 +404,6 @@ class AttentionGuidedCAMLLaVA(AttentionGuidedCAM):
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
 class AttentionGuidedCAMChartGemma(AttentionGuidedCAM):
     def __init__(self, model, target_layers):
         self.target_layers = target_layers
@@ -489,7 +450,6 @@ class AttentionGuidedCAMChartGemma(AttentionGuidedCAM):
         self.model.zero_grad()
         # print(outputs_raw)
         loss = outputs_raw.logits.max(dim=-1).values.sum()
-
         loss.backward()
 
         # get image masks
@@ -507,75 +467,54 @@ class AttentionGuidedCAMChartGemma(AttentionGuidedCAM):
         # Aggregate activations and gradients from ALL layers
         self.activations = [layer.get_attn_map() for layer in self.target_layers]
         self.gradients = [layer.get_attn_gradients() for layer in self.target_layers]
-        cam_sum = None
-        # Ver 1
-        # for act, grad in zip(self.activations, self.gradients):
-        #     # act = torch.sigmoid(act)
-        #     print("act:", act)
-        #     print(len(act))
-        #     print("act_shape:", act.shape)
-        #     # print("act1_shape:", act[1].shape)
-            
-        #     act = F.relu(act.mean(dim=1))
-
-
-        #     # Compute mean of gradients
-        #     print("grad:", grad)
-        #     print(len(grad))
-        #     print("grad_shape:", grad.shape)
-        #     grad_weights = grad.mean(dim=1)
-
-        #     print("act shape", act.shape)
-        #     print("grad_weights shape", grad_weights.shape)
-
-        #     cam = act * grad_weights
-        #     # cam = act
-        #     print(cam.shape)
-
-        #     # Sum across all layers
-        #     if cam_sum is None:
-        #         cam_sum = cam
-        #     else:
-        #         cam_sum += cam  
+        print(f"layers shape: {len(self.target_layers)}")
+        print("activations & gradients shape", len(self.activations), len(self.gradients))
 
+        cams = []
+        
         # Ver 2
         for act, grad in zip(self.activations, self.gradients):
 
             print("act shape", act.shape)
             print("grad shape", grad.shape)
-
+            
             grad = F.relu(grad)
 
             cam = act * grad # shape: [1, heads, seq_len, seq_len]
             cam = cam.sum(dim=1) # shape: [1, seq_len, seq_len]
+            cam = cam.to(torch.float32).detach().cpu()
+            cams.append(cam)
 
-            # Sum across all layers
-            if cam_sum is None:
-                cam_sum = cam
-            else:
-                cam_sum += cam 
+        # cam_sum = F.relu(cam_sum)
+        # cam_sum = cam_sum.to(torch.float32)
 
-        cam_sum = F.relu(cam_sum)
-        cam_sum = cam_sum.to(torch.float32)
-
-        # cam_sum shape: [1, seq_len, seq_len]
+        # cams shape: [layers, 1, seq_len, seq_len]
         cam_sum_lst = []
-        cam_sum_raw = cam_sum
+
         start_idx = last + 1
-        for i in range(start_idx, cam_sum_raw.shape[1]):
-            cam_sum = cam_sum_raw[0, i, :] # shape: [1: seq_len]
-            # cam_sum_min = cam_sum.min()
-            # cam_sum_max = cam_sum.max()
-            # cam_sum = (cam_sum - cam_sum_min) / (cam_sum_max - cam_sum_min)
-            cam_sum = cam_sum[image_mask].unsqueeze(0) # shape: [1, 1024]
-            print("cam_sum shape: ", cam_sum.shape)
-            num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
-            grid_size = int(num_patches ** 0.5)
-            print(f"Detected grid size: {grid_size}x{grid_size}")
+        for i in range(start_idx, cams[0].shape[1]):
+            cam_sum = None
+            for layer, cam_l in enumerate(cams):
+                cam_l_i = cam_l[0, i, :] # shape: [1: seq_len]
 
-            # Fix the reshaping step dynamically
-            
-            cam_sum = cam_sum.view(grid_size, grid_size)
+                cam_l_i = cam_l_i[image_mask].unsqueeze(0) # shape: [1, img_seq_len]
+                # print(f"layer: {layer}, token index: {i}")
+                # print("cam_sum shape: ", cam_l_i.shape)
+                num_patches = cam_l_i.shape[-1]  # Last dimension of CAM output
+                grid_size = int(num_patches ** 0.5)
+                # print(f"Detected grid size: {grid_size}x{grid_size}")
+
+                # Fix the reshaping step dynamically
+                cam_reshaped = cam_l_i.view(grid_size, grid_size)
+                # print(f"max: {cam_reshaped.max()}, min: {cam_reshaped.min()}")
+                cam_normalized = (cam_reshaped - cam_reshaped.min()) / (cam_reshaped.max() - cam_reshaped.min())
+                if cam_sum == None:
+                    cam_sum = cam_normalized
+                else:
+                    cam_sum += cam_normalized
+                # print(f"normalized: max: {cam_normalized.max()}, min: {cam_normalized.min()}")
+
+            # print(f"sum: max: {cam_sum.max()}, min: {cam_sum.min()}")
             cam_sum = (cam_sum - cam_sum.min()) / (cam_sum.max() - cam_sum.min())
             cam_sum_lst.append(cam_sum)
 
@@ -604,7 +543,7 @@ def generate_gradcam(
     alpha=0.5, 
     colormap=cv2.COLORMAP_JET, 
     aggregation='mean',
-    normalize=True
+    normalize=False
 ):
     """
     Generates a Grad-CAM heatmap overlay on top of the input image.