updated filtering, add F= tab

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

app.py

@@ -6,6 +6,9 @@ import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 from io import StringIO
+from typing import Dict, List, Optional
+from dataclasses import dataclass, field
+from copy import deepcopy
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -39,20 +42,6 @@ from src.submission.submit import add_new_eval
 
 from src.about import TasksMib_Subgraph
 
-
-
-
-
-
-
-
-
-
-from gradio_leaderboard import SelectColumns, Leaderboard
-import pandas as pd
-from typing import List, Dict, Union, Optional, Any
-from dataclasses import fields
-
 # class SmartSelectColumns(SelectColumns):
 #     """
 #     Enhanced SelectColumns component with basic filtering functionality.
@@ -130,9 +119,6 @@ from dataclasses import fields
             
 #         return super().update(value)
 
-
-
-
 from gradio_leaderboard import SelectColumns, Leaderboard
 import pandas as pd
 from typing import List, Dict, Optional
@@ -194,10 +180,60 @@ class SmartSelectColumns(SelectColumns):
         return filtered_groups
 
 
-
-
-
-
+import re
+@dataclass
+class SubstringSelectColumns(SelectColumns):
+    """
+    Extends SelectColumns to support filtering columns by predefined substrings.
+    When a substring is selected, all columns containing that substring will be selected.
+    """
+    substring_groups: Dict[str, List[str]] = field(default_factory=dict)
+    selected_substrings: List[str] = field(default_factory=list)
+    
+    def __post_init__(self):
+        # Ensure default_selection is a list
+        if self.default_selection is None:
+            self.default_selection = []
+            
+        # Build reverse mapping of column to substrings
+        self.column_to_substrings = {}
+        for substring, patterns in self.substring_groups.items():
+            for pattern in patterns:
+                # Convert glob-style patterns to regex
+                regex = re.compile(pattern.replace('*', '.*'))
+                # Find matching columns in default_selection
+                for col in self.default_selection:
+                    if regex.search(col):
+                        if col not in self.column_to_substrings:
+                            self.column_to_substrings[col] = []
+                        self.column_to_substrings[col].append(substring)
+        
+        # Apply initial substring selections
+        if self.selected_substrings:
+            self.update_selection_from_substrings()
+    
+    def update_selection_from_substrings(self) -> List[str]:
+        """
+        Updates the column selection based on selected substrings.
+        Returns the new list of selected columns.
+        """
+        selected_columns = self.cant_deselect.copy()
+        
+        # If no substrings selected, show all columns
+        if not self.selected_substrings:
+            selected_columns.extend([
+                col for col in self.default_selection 
+                if col not in self.cant_deselect
+            ])
+            return selected_columns
+        
+        # Add columns that match any selected substring
+        for col, substrings in self.column_to_substrings.items():
+            if any(s in self.selected_substrings for s in substrings):
+                if col not in selected_columns:
+                    selected_columns.append(col)
+                    
+        return selected_columns
 
 
 
@@ -237,7 +273,9 @@ except Exception:
 
 
 
-LEADERBOARD_DF_MIB_SUBGRAPH = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_SUBGRAPH, BENCHMARK_COLS_MIB_SUBGRAPH)
+LEADERBOARD_DF_MIB_SUBGRAPH_FPL = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_SUBGRAPH, BENCHMARK_COLS_MIB_SUBGRAPH)
+LEADERBOARD_DF_MIB_SUBGRAPH_FEQ = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_SUBGRAPH, BENCHMARK_COLS_MIB_SUBGRAPH,
+                                                                  metric_type="F=")
 
 # LEADERBOARD_DF_MIB_CAUSALGRAPH = get_leaderboard_df_mib_causalgraph(EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_CAUSALGRAPH, BENCHMARK_COLS_MIB_CAUSALGRAPH)
 # In app.py, modify the LEADERBOARD initialization
@@ -364,23 +402,23 @@ def init_leaderboard_mib_subgraph(dataframe, track):
         "qwen2_5": "Qwen-2.5",
         "gpt2": "GPT-2",
         "gemma2": "Gemma-2",
-        "llama3": "LLaMA-3"
+        "llama3": "Llama-3.1"
     }
 
     benchmark_mapping = {
         "ioi": "IOI",
         "mcqa": "MCQA",
-        "arithmetic_addition": "Arithmetic Addition",
-        "arithmetic_subtraction": "Arithmetic Subtraction",
-        "arc_easy": "ARC Easy",
-        "arc_challenge": "ARC Challenge"
+        "arithmetic_addition": "Arithmetic (+)",
+        "arithmetic_subtraction": "Arithmetic (-)",
+        "arc_easy": "ARC (Easy)",
+        "arc_challenge": "ARC (Challenge)"
     }
 
     display_mapping = {}
     for task in TasksMib_Subgraph:
         for model in task.value.models:
             field_name = f"{task.value.benchmark}_{model}"
-            display_name = f"{benchmark_mapping[task.value.benchmark]}( {model_name_mapping[model]} )"
+            display_name = f"{benchmark_mapping[task.value.benchmark]} - {model_name_mapping[model]}"
             display_mapping[field_name] = display_name
     
 
@@ -423,18 +461,19 @@ def init_leaderboard_mib_subgraph(dataframe, track):
     # all_columns = [c.name for c in fields(AutoEvalColumn_mib_subgraph)]
     all_columns = renamed_df.columns.tolist()
 
+
     # Original code 
     return Leaderboard(
         value=renamed_df,  # Use DataFrame with display names
         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
-        select_columns=SelectColumns(
-            default_selection=all_columns,  # Now contains display names
-            label="Select Results:"
-        ),
+        # select_columns=SelectColumns(
+        #     default_selection=all_columns,  # Now contains display names
+        #     label="Filter Results:",
+        # ),
         search_columns=["Method"],
-        hide_columns=[],
+        hide_columns=["eval_name"],
         interactive=False,
-    )
+    ), renamed_df
     
 
 
@@ -685,6 +724,32 @@ def process_json(temp_file):
     return data
 
 
+# Define the preset substrings for filtering
+PRESET_SUBSTRINGS = ["IOI", "MCQA", "Arithmetic", "ARC", "GPT-2", "Qwen-2.5", "Gemma-2", "Llama-3.1"]
+
+def filter_columns_by_substrings(dataframe: pd.DataFrame, selected_substrings: List[str]) -> pd.DataFrame:
+    """
+    Filter columns based on the selected substrings.
+    """
+    original_dataframe = deepcopy(dataframe)
+    if not selected_substrings:
+        return dataframe  # No filtering if no substrings are selected
+
+    # Filter columns that contain any of the selected substrings
+    filtered_columns = [
+        col for col in dataframe.columns
+        if any(sub.lower() in col.lower() for sub in selected_substrings)
+        or col == "Method"
+    ]
+    return dataframe[filtered_columns]
+
+def update_leaderboard(dataframe: pd.DataFrame, selected_substrings: List[str]):
+    """
+    Update the leaderboard based on the selected substrings.
+    """
+    filtered_dataframe = filter_columns_by_substrings(dataframe, selected_substrings)
+    return filtered_dataframe
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -709,14 +774,51 @@ with demo:
         # with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
         #     leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
         with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
-            # Add description for filters
-            gr.Markdown("""
-            ### Filtering Options
-            Use the dropdown menus below to filter results by specific tasks or models. 
-            You can combine filters to see specific task-model combinations.
-            """)
-            leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
-            print(f"Leaderboard is {leaderboard}")
+            with gr.Tabs() as subgraph_tabs:
+                with gr.TabItem("F+", id=0):
+                    # Add description for filters
+                    gr.Markdown("""
+                    ### Filtering Options
+                    Use the dropdown menus below to filter results by specific tasks or models. 
+                    You can combine filters to see specific task-model combinations.
+                    """)
+                    # CheckboxGroup for selecting substrings
+                    substring_checkbox = gr.CheckboxGroup(
+                        choices=PRESET_SUBSTRINGS,
+                        label="Filter results:",
+                        value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    )
+                    leaderboard, data = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH_FPL, "Subgraph")
+                    original_leaderboard = gr.State(value=data)
+                    # Update the leaderboard when the user selects/deselects substrings
+                    substring_checkbox.change(
+                        fn=update_leaderboard,
+                        inputs=[original_leaderboard, substring_checkbox],
+                        outputs=leaderboard
+                    )
+                    print(f"Leaderboard is {leaderboard}")
+                with gr.TabItem("F=", id=1):
+                    # Add description for filters
+                    gr.Markdown("""
+                    ### Filtering Options
+                    Use the dropdown menus below to filter results by specific tasks or models. 
+                    You can combine filters to see specific task-model combinations.
+                    """)
+                    # CheckboxGroup for selecting substrings
+                    substring_checkbox = gr.CheckboxGroup(
+                        choices=PRESET_SUBSTRINGS,
+                        label="Filter results:",
+                        value=PRESET_SUBSTRINGS,  # Default to all substrings selected
+                    )
+                    leaderboard, data = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH_FEQ, "Subgraph")
+                    original_leaderboard = gr.State(value=data)
+                    # Update the leaderboard when the user selects/deselects substrings
+                    substring_checkbox.change(
+                        fn=update_leaderboard,
+                        inputs=[original_leaderboard, substring_checkbox],
+                        outputs=leaderboard
+                    )
+                    print(f"Leaderboard is {leaderboard}")
     
         # Then modify the Causal Graph tab section
         with gr.TabItem("Causal Graph", elem_id="causalgraph", id=1):

app.pyi

@@ -0,0 +1,867 @@
+import json
+import gzip
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+from io import StringIO
+from dataclasses import dataclass, field
+
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    BENCHMARK_COLS,
+    BENCHMARK_COLS_MULTIMODAL,
+    BENCHMARK_COLS_MIB_SUBGRAPH,
+    BENCHMARK_COLS_MIB_CAUSALGRAPH,
+    COLS,
+    COLS_MIB_SUBGRAPH,
+    COLS_MIB_CAUSALGRAPH,
+    COLS_MULTIMODAL,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    AutoEvalColumn_mib_subgraph,
+    AutoEvalColumn_mib_causalgraph,
+    fields,
+)
+from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, REPO_ID, TOKEN, RESULTS_REPO_MIB_SUBGRAPH, EVAL_RESULTS_MIB_SUBGRAPH_PATH, RESULTS_REPO_MIB_CAUSALGRAPH, EVAL_RESULTS_MIB_CAUSALGRAPH_PATH
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_leaderboard_df_mib_subgraph, get_leaderboard_df_mib_causalgraph
+from src.submission.submit import add_new_eval
+
+
+from src.about import TasksMib_Subgraph
+
+# class SmartSelectColumns(SelectColumns):
+#     """
+#     Enhanced SelectColumns component with basic filtering functionality.
+#     """
+#     def __init__(
+#         self,
+#         benchmark_keywords: Optional[List[str]] = None,
+#         model_keywords: Optional[List[str]] = None,
+#         initial_selected: Optional[List[str]] = None,
+#         **kwargs
+#     ):
+#         """
+#         Initialize SmartSelectColumns with minimal configuration.
+        
+#         Args:
+#             benchmark_keywords: List of benchmark names to filter by
+#             model_keywords: List of model names to filter by
+#             initial_selected: List of columns to show initially
+#         """
+#         super().__init__(**kwargs)
+#         self.benchmark_keywords = benchmark_keywords or []
+#         self.model_keywords = model_keywords or []
+#         self.initial_selected = initial_selected or []
+
+#     def get_filtered_groups(self, df: pd.DataFrame) -> Dict[str, List[str]]:
+#         """
+#         Create column groups based on simple substring matching.
+#         """
+#         filtered_groups = {}
+        
+#         # Create benchmark groups
+#         for benchmark in self.benchmark_keywords:
+#             matching_cols = [
+#                 col for col in df.columns 
+#                 if benchmark in col.lower()
+#             ]
+#             if matching_cols:
+#                 group_name = f"Benchmark group for {benchmark}"
+#                 filtered_groups[group_name] = matching_cols
+        
+#         # Create model groups
+#         for model in self.model_keywords:
+#             matching_cols = [
+#                 col for col in df.columns 
+#                 if model in col.lower()
+#             ]
+#             if matching_cols:
+#                 group_name = f"Model group for {model}"
+#                 filtered_groups[group_name] = matching_cols
+        
+#         return filtered_groups
+
+#     def update(
+#         self, 
+#         value: Union[pd.DataFrame, Dict[str, List[str]], Any]
+#     ) -> Dict:
+#         """Update component with new values."""
+#         if isinstance(value, pd.DataFrame):
+#             choices = list(value.columns)
+#             selected = self.initial_selected if self.initial_selected else choices
+#             filtered_cols = self.get_filtered_groups(value)
+            
+#             return {
+#                 "choices": choices,
+#                 "value": selected,
+#                 "filtered_cols": filtered_cols
+#             }
+        
+#         if hasattr(value, '__dataclass_fields__'):
+#             field_names = [field.name for field in fields(value)]
+#             return {
+#                 "choices": field_names,
+#                 "value": self.initial_selected if self.initial_selected else field_names
+#             }
+            
+#         return super().update(value)
+
+from gradio.events import Dependency
+
+class ModifiedLeaderboard(Leaderboard):
+    """Extends Leaderboard to support substring-based column filtering"""
+    
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        
+        # Process substring groups if they exist
+        if (isinstance(self.select_columns_config, SelectColumns) and 
+            self.select_columns_config.substring_groups):
+            
+            self.process_substring_groups()
+    
+    def process_substring_groups(self):
+        """Processes substring groups to add them to the selectable columns"""
+        groups = self.select_columns_config.substring_groups
+        if not groups:
+            return
+            
+        # Create a mapping of group name to matching columns
+        group_to_columns = {}
+        for group_name, patterns in groups.groups.items():
+            matching_cols = set()
+            for pattern in patterns:
+                regex = re.compile(pattern.replace('*', '.*'))
+                matching_cols.update(
+                    col for col in self.headers 
+                    if regex.search(col)
+                )
+            if matching_cols:
+                group_to_columns[group_name] = list(matching_cols)
+        
+        # Add groups to the headers and update column selection logic
+        self.group_to_columns = group_to_columns
+        self.original_headers = self.headers.copy()
+        
+        # Add group names to the start of headers
+        self.headers = list(group_to_columns.keys()) + self.original_headers
+        
+        # Update default selection to include groups
+        if self.select_columns_config.default_selection:
+            self.select_columns_config.default_selection = (
+                list(group_to_columns.keys()) + 
+                self.select_columns_config.default_selection
+            )
+
+    def preprocess(self, payload):
+        """Override preprocess to handle group selection"""
+        df = super().preprocess(payload)
+        
+        # If we don't have substring groups, return normally
+        if not hasattr(self, 'group_to_columns'):
+            return df
+            
+        # Process group selections
+        selected_columns = set()
+        for column in payload.headers:
+            if column in self.group_to_columns:
+                # If a group is selected, add all its columns
+                selected_columns.update(self.group_to_columns[column])
+            elif column in self.original_headers:
+                # Add individually selected columns
+                selected_columns.add(column)
+        
+        # Return DataFrame with only selected columns
+        return df[list(selected_columns)]
+    from typing import Callable, Literal, Sequence, Any, TYPE_CHECKING
+    from gradio.blocks import Block
+    if TYPE_CHECKING:
+        from gradio.components import Timer
+
+from gradio_leaderboard import SelectColumns, Leaderboard
+import pandas as pd
+from typing import List, Dict, Optional
+from dataclasses import fields
+
+class SmartSelectColumns(SelectColumns):
+    """
+    Enhanced SelectColumns component matching exact original parameters.
+    """
+    def __init__(
+        self,
+        benchmark_keywords: Optional[List[str]] = None,
+        model_keywords: Optional[List[str]] = None,
+        initial_selected: Optional[List[str]] = None,
+        label: Optional[str] = None,
+        show_label: bool = True,
+        info: Optional[str] = None,
+        allow: bool = True
+    ):
+        # Match exact parameters from working SelectColumns
+        super().__init__(
+            default_selection=initial_selected or [],
+            cant_deselect=[],
+            allow=allow,
+            label=label,
+            show_label=show_label,
+            info=info
+        )
+        
+        self.benchmark_keywords = benchmark_keywords or []
+        self.model_keywords = model_keywords or []
+        
+        # Store groups for later use
+        self._groups = {}
+    
+    def get_filtered_groups(self, columns: List[str]) -> Dict[str, List[str]]:
+        """Get column groups based on keywords."""
+        filtered_groups = {}
+        
+        # Add benchmark groups
+        for benchmark in self.benchmark_keywords:
+            matching_cols = [
+                col for col in columns 
+                if benchmark in col.lower()
+            ]
+            if matching_cols:
+                filtered_groups[f"Benchmark group for {benchmark}"] = matching_cols
+        
+        # Add model groups
+        for model in self.model_keywords:
+            matching_cols = [
+                col for col in columns 
+                if model in col.lower()
+            ]
+            if matching_cols:
+                filtered_groups[f"Model group for {model}"] = matching_cols
+                
+        self._groups = filtered_groups
+        return filtered_groups
+
+
+import re
+@dataclass
+class SubstringSelectColumns(SelectColumns):
+    """
+    Extends SelectColumns to support filtering columns by predefined substrings.
+    When a substring is selected, all columns containing that substring will be selected.
+    """
+    substring_groups: Dict[str, List[str]] = field(default_factory=dict)
+    selected_substrings: List[str] = field(default_factory=list)
+    
+    def __post_init__(self):
+        # Ensure default_selection is a list
+        if self.default_selection is None:
+            self.default_selection = []
+            
+        # Build reverse mapping of column to substrings
+        self.column_to_substrings = {}
+        for substring, patterns in self.substring_groups.items():
+            for pattern in patterns:
+                # Convert glob-style patterns to regex
+                regex = re.compile(pattern.replace('*', '.*'))
+                # Find matching columns in default_selection
+                for col in self.default_selection:
+                    if regex.search(col):
+                        if col not in self.column_to_substrings:
+                            self.column_to_substrings[col] = []
+                        self.column_to_substrings[col].append(substring)
+        
+        # Apply initial substring selections
+        if self.selected_substrings:
+            self.update_selection_from_substrings()
+    
+    def update_selection_from_substrings(self) -> List[str]:
+        """
+        Updates the column selection based on selected substrings.
+        Returns the new list of selected columns.
+        """
+        selected_columns = self.cant_deselect.copy()
+        
+        # If no substrings selected, show all columns
+        if not self.selected_substrings:
+            selected_columns.extend([
+                col for col in self.default_selection 
+                if col not in self.cant_deselect
+            ])
+            return selected_columns
+        
+        # Add columns that match any selected substring
+        for col, substrings in self.column_to_substrings.items():
+            if any(s in self.selected_substrings for s in substrings):
+                if col not in selected_columns:
+                    selected_columns.append(col)
+                    
+        return selected_columns
+
+
+
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+
+
+### Space initialisation
+try:
+    # print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+
+try:
+    # print(RESULTS_REPO_MIB_SUBGRAPH)
+    snapshot_download(
+        repo_id=RESULTS_REPO_MIB_SUBGRAPH, local_dir=EVAL_RESULTS_MIB_SUBGRAPH_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+
+try:
+    # print(RESULTS_REPO_MIB_CAUSALGRAPH)
+    snapshot_download(
+        repo_id=RESULTS_REPO_MIB_CAUSALGRAPH, local_dir=EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+
+
+LEADERBOARD_DF_MIB_SUBGRAPH = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_SUBGRAPH, BENCHMARK_COLS_MIB_SUBGRAPH)
+
+# LEADERBOARD_DF_MIB_CAUSALGRAPH = get_leaderboard_df_mib_causalgraph(EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_CAUSALGRAPH, BENCHMARK_COLS_MIB_CAUSALGRAPH)
+# In app.py, modify the LEADERBOARD initialization
+LEADERBOARD_DF_MIB_CAUSALGRAPH_DETAILED, LEADERBOARD_DF_MIB_CAUSALGRAPH_AGGREGATED, LEADERBOARD_DF_MIB_CAUSALGRAPH_AVERAGED = get_leaderboard_df_mib_causalgraph(
+    EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, 
+    EVAL_REQUESTS_PATH, 
+    COLS_MIB_CAUSALGRAPH, 
+    BENCHMARK_COLS_MIB_CAUSALGRAPH
+)
+
+
+# LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+# LEADERBOARD_DF_MULTIMODAL = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS_MULTIMODAL, BENCHMARK_COLS_MULTIMODAL)
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+
+
+
+
+# def init_leaderboard_mib_subgraph(dataframe, track):
+#     # print(f"init_leaderboard_mib: dataframe head before loc is {dataframe.head()}\n")
+    
+#     if dataframe is None or dataframe.empty:
+#         raise ValueError("Leaderboard DataFrame is empty or None.")
+
+#     # filter for correct track
+#     # dataframe = dataframe.loc[dataframe["Track"] == track]
+
+#     # print(f"init_leaderboard_mib: dataframe head after loc is {dataframe.head()}\n")
+    
+#     return Leaderboard(
+#         value=dataframe,
+#         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
+#         select_columns=SelectColumns(
+#             default_selection=[c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.displayed_by_default],
+#             cant_deselect=[c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.never_hidden],
+#             label="Select Columns to Display:",
+#         ),
+#         search_columns=["Method"],  # Changed from AutoEvalColumn_mib_subgraph.model.name to "Method"
+#         hide_columns=[c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.hidden],
+#         bool_checkboxgroup_label="Hide models",
+#         interactive=False,
+#     )
+
+
+
+
+
+# def init_leaderboard_mib_subgraph(dataframe, track):
+#     """Initialize the subgraph leaderboard with grouped column selection by benchmark."""
+#     if dataframe is None or dataframe.empty:
+#         raise ValueError("Leaderboard DataFrame is empty or None.")
+
+#     print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
+    
+#     # Create groups of columns by benchmark
+#     benchmark_groups = []
+    
+#     # For each benchmark in our TasksMib_Subgraph enum...
+#     for task in TasksMib_Subgraph:
+#         benchmark = task.value.benchmark
+#         # Get all valid columns for this benchmark's models
+#         benchmark_cols = [
+#             f"{benchmark}_{model}"
+#             for model in task.value.models
+#             if f"{benchmark}_{model}" in dataframe.columns
+#         ]
+#         if benchmark_cols:  # Only add if we have valid columns
+#             benchmark_groups.append(benchmark_cols)
+#             print(f"\nBenchmark group for {benchmark}:", benchmark_cols)
+
+#     # Create model groups as well
+#     model_groups = []
+#     all_models = list(set(model for task in TasksMib_Subgraph for model in task.value.models))
+    
+#     # For each unique model...
+#     for model in all_models:
+#         # Get all valid columns for this model across benchmarks
+#         model_cols = [
+#             f"{task.value.benchmark}_{model}"
+#             for task in TasksMib_Subgraph
+#             if model in task.value.models
+#             and f"{task.value.benchmark}_{model}" in dataframe.columns
+#         ]
+#         if model_cols:  # Only add if we have valid columns
+#             model_groups.append(model_cols)
+#             print(f"\nModel group for {model}:", model_cols)
+
+#     # Combine all groups
+#     all_groups = benchmark_groups + model_groups
+    
+#     # Flatten groups for default selection (show everything initially)
+#     all_columns = [col for group in all_groups for col in group]
+#     print("\nAll available columns:", all_columns)
+
+#     return Leaderboard(
+#         value=dataframe,
+#         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
+#         select_columns=SelectColumns(
+#             default_selection=all_columns,  # Show all columns initially
+#             label="Select Results:"
+#         ),
+#         search_columns=["Method"],
+#         hide_columns=[],
+#         interactive=False,
+#     )
+
+
+def init_leaderboard_mib_subgraph(dataframe, track):
+    """Initialize the subgraph leaderboard with display names for better readability."""
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
+    
+    # First, create our display name mapping
+    # This is like creating a translation dictionary between internal names and display names
+    model_name_mapping = {
+        "qwen2_5": "Qwen-2.5",
+        "gpt2": "GPT-2",
+        "gemma2": "Gemma-2",
+        "llama3": "Llama-3.1"
+    }
+
+    benchmark_mapping = {
+        "ioi": "IOI",
+        "mcqa": "MCQA",
+        "arithmetic_addition": "Arithmetic (+)",
+        "arithmetic_subtraction": "Arithmetic (-)",
+        "arc_easy": "ARC (Easy)",
+        "arc_challenge": "ARC (Challenge)"
+    }
+
+    display_mapping = {}
+    for task in TasksMib_Subgraph:
+        for model in task.value.models:
+            field_name = f"{task.value.benchmark}_{model}"
+            display_name = f"{benchmark_mapping[task.value.benchmark]} - {model_name_mapping[model]}"
+            display_mapping[field_name] = display_name
+    
+
+    # Now when creating benchmark groups, we'll use display names
+    benchmark_groups = []
+    for task in TasksMib_Subgraph:
+        benchmark = task.value.benchmark
+        benchmark_cols = [
+            display_mapping[f"{benchmark}_{model}"]  # Use display name from our mapping
+            for model in task.value.models
+            if f"{benchmark}_{model}" in dataframe.columns
+        ]
+        if benchmark_cols:
+            benchmark_groups.append(benchmark_cols)
+            print(f"\nBenchmark group for {benchmark}:", benchmark_cols)
+
+    # Similarly for model groups
+    model_groups = []
+    all_models = list(set(model for task in TasksMib_Subgraph for model in task.value.models))
+    
+    for model in all_models:
+        model_cols = [
+            display_mapping[f"{task.value.benchmark}_{model}"]  # Use display name
+            for task in TasksMib_Subgraph
+            if model in task.value.models
+            and f"{task.value.benchmark}_{model}" in dataframe.columns
+        ]
+        if model_cols:
+            model_groups.append(model_cols)
+            print(f"\nModel group for {model}:", model_cols)
+
+    # Combine all groups using display names
+    all_groups = benchmark_groups + model_groups
+    all_columns = [col for group in all_groups for col in group]
+
+    # Important: We need to rename our DataFrame columns to match display names
+    
+    renamed_df = dataframe.rename(columns=display_mapping)
+    # all_columns = [c.name for c in fields(AutoEvalColumn_mib_subgraph) if c.displayed_by_default]
+    # all_columns = [c.name for c in fields(AutoEvalColumn_mib_subgraph)]
+    all_columns = renamed_df.columns.tolist()
+
+    print(benchmark_groups)
+    print(model_groups)
+    filter_groups = {"ioi": "*IOI*",
+                     "llama": "*Llama*"}
+
+
+    # Original code 
+    return ModifiedLeaderboard(
+        value=renamed_df,  # Use DataFrame with display names
+        datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
+        select_columns=SubstringSelectColumns(
+            substring_groups=filter_groups,
+            default_selection=all_columns,  # Now contains display names
+            label="Filter Results:",
+            allow=True
+        ),
+        search_columns=["Method"],
+        hide_columns=[],
+        interactive=False,
+    )
+    
+
+
+
+    # # Complete column groups for both benchmarks and models
+    # # Define keywords for filtering
+    # benchmark_keywords = ["ioi", "mcqa", "arithmetic_addition", "arithmetic_subtraction", "arc_easy", "arc_challenge"]
+    # model_keywords = ["qwen2_5", "gpt2", "gemma2", "llama3"]
+
+    # # Optional: Define display names
+    # mappings = {
+    #     "ioi_llama3": "IOI (LLaMA-3)",
+    #     "ioi_qwen2_5": "IOI (Qwen-2.5)",
+    #     "ioi_gpt2": "IOI (GPT-2)",
+    #     "ioi_gemma2": "IOI (Gemma-2)",
+    #     "mcqa_llama3": "MCQA (LLaMA-3)",
+    #     "mcqa_qwen2_5": "MCQA (Qwen-2.5)",
+    #     "mcqa_gemma2": "MCQA (Gemma-2)",
+    #     "arithmetic_addition_llama3": "Arithmetic Addition (LLaMA-3)",
+    #     "arithmetic_subtraction_llama3": "Arithmetic Subtraction (LLaMA-3)",
+    #     "arc_easy_llama3": "ARC Easy (LLaMA-3)",
+    #     "arc_easy_gemma2": "ARC Easy (Gemma-2)",
+    #     "arc_challenge_llama3": "ARC Challenge (LLaMA-3)",
+    #     "eval_name": "Evaluation Name",
+    #     "Method": "Method",
+    #     "Average": "Average Score"
+    # }
+    # # mappings = {}
+
+    # # Create SmartSelectColumns instance
+    # smart_columns = SmartSelectColumns(
+    #     benchmark_keywords=benchmark_keywords,
+    #     model_keywords=model_keywords,
+    #     column_mapping=mappings,
+    #     initial_selected=["Method", "Average"]
+    # )
+
+    # print("\nDebugging DataFrame columns:", renamed_df.columns.tolist())
+
+    # # Create Leaderboard
+    # leaderboard = Leaderboard(
+    #     value=renamed_df,
+    #     datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
+    #     select_columns=smart_columns,
+    #     search_columns=["Method"],
+    #     hide_columns=[],
+    #     interactive=False
+    # )
+    # print(f"Successfully created leaderboard.")
+    # return leaderboard
+
+    # print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
+
+    # # Define simple keywords for filtering
+    # benchmark_keywords = ["ioi", "mcqa", "arithmetic", "arc"]
+    # model_keywords = ["qwen2_5", "gpt2", "gemma2", "llama3"]
+
+    # # Create SmartSelectColumns instance with exact same parameters as working version
+    # smart_columns = SmartSelectColumns(
+    #     benchmark_keywords=benchmark_keywords,
+    #     model_keywords=model_keywords,
+    #     initial_selected=["Method", "Average"],
+    #     allow=True,
+    #     label=None,
+    #     show_label=True,
+    #     info=None
+    # )
+
+    # try:
+    #     print("\nCreating leaderboard...")
+    #     # Get groups before creating leaderboard
+    #     smart_columns.get_filtered_groups(dataframe.columns)
+        
+    #     leaderboard = Leaderboard(
+    #         value=dataframe,
+    #         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
+    #         select_columns=smart_columns,
+    #         search_columns=["Method"],
+    #         hide_columns=[],
+    #         interactive=False
+    #     )
+    #     print("Leaderboard created successfully")
+    #     return leaderboard
+        
+    # except Exception as e:
+    #     print("Error creating leaderboard:", str(e))
+    #     raise
+
+
+
+
+
+
+# def init_leaderboard_mib_subgraph(dataframe, track):
+#     """Initialize the subgraph leaderboard with group-based column selection."""
+#     if dataframe is None or dataframe.empty:
+#         raise ValueError("Leaderboard DataFrame is empty or None.")
+
+#     print("\nDebugging DataFrame columns:", dataframe.columns.tolist())
+    
+#     # Create selection mapping for benchmark groups
+#     selection_mapping = {}
+    
+#     # Create benchmark groups with descriptive names
+#     for task in TasksMib_Subgraph:
+#         benchmark = task.value.benchmark
+#         # Get all columns for this benchmark's models
+#         benchmark_cols = [
+#             f"{benchmark}_{model}"
+#             for model in task.value.models
+#             if f"{benchmark}_{model}" in dataframe.columns
+#         ]
+#         if benchmark_cols:
+#             # Use a descriptive group name as the key
+#             group_name = f"Benchmark: {benchmark.upper()}"
+#             selection_mapping[group_name] = benchmark_cols
+#             print(f"\n{group_name} maps to:", benchmark_cols)
+
+#     # Create model groups with descriptive names
+#     all_models = list(set(model for task in TasksMib_Subgraph for model in task.value.models))
+#     for model in all_models:
+#         # Get all columns for this model across benchmarks
+#         model_cols = [
+#             f"{task.value.benchmark}_{model}"
+#             for task in TasksMib_Subgraph
+#             if model in task.value.models
+#             and f"{task.value.benchmark}_{model}" in dataframe.columns
+#         ]
+#         if model_cols:
+#             # Use a descriptive group name as the key
+#             group_name = f"Model: {model}"
+#             selection_mapping[group_name] = model_cols
+#             print(f"\n{group_name} maps to:", model_cols)
+
+#     # The selection options are the group names
+#     selection_options = list(selection_mapping.keys())
+#     print("\nSelection options:", selection_options)
+
+#     return Leaderboard(
+#         value=dataframe,
+#         datatype=[c.type for c in fields(AutoEvalColumn_mib_subgraph)],
+#         select_columns=SelectColumns(
+#             default_selection=selection_options,  # Show all groups by default
+#             label="Select Benchmark or Model Groups:"
+#         ),
+#         search_columns=["Method"],
+#         hide_columns=[],
+#         interactive=False,
+#     )
+
+
+
+
+
+
+
+
+# def init_leaderboard_mib_causalgraph(dataframe, track):
+#     # print("Debugging column issues:")
+#     # print("\nActual DataFrame columns:")
+#     # print(dataframe.columns.tolist())
+    
+#     # print("\nExpected columns for Leaderboard:")
+#     expected_cols = [c.name for c in fields(AutoEvalColumn_mib_causalgraph)]
+#     # print(expected_cols)
+    
+#     # print("\nMissing columns:")
+#     missing_cols = [col for col in expected_cols if col not in dataframe.columns]
+#     # print(missing_cols)
+    
+#     # print("\nSample of DataFrame content:")
+#     # print(dataframe.head().to_string())
+
+#     return Leaderboard(
+#         value=dataframe,
+#         datatype=[c.type for c in fields(AutoEvalColumn_mib_causalgraph)],
+#         select_columns=SelectColumns(
+#             default_selection=[c.name for c in fields(AutoEvalColumn_mib_causalgraph) if c.displayed_by_default],
+#             cant_deselect=[c.name for c in fields(AutoEvalColumn_mib_causalgraph) if c.never_hidden],
+#             label="Select Columns to Display:",
+#         ),
+#         search_columns=["Method"],
+#         hide_columns=[c.name for c in fields(AutoEvalColumn_mib_causalgraph) if c.hidden],
+#         bool_checkboxgroup_label="Hide models",
+#         interactive=False,
+#     )
+
+def init_leaderboard_mib_causalgraph(dataframe, track):
+    # print("Debugging column issues:")
+    # print("\nActual DataFrame columns:")
+    # print(dataframe.columns.tolist())
+    
+    # Create only necessary columns
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn_mib_causalgraph)],
+        select_columns=SelectColumns(
+            default_selection=["Method"],  # Start with just Method column
+            cant_deselect=["Method"],      # Method column should always be visible
+            label="Select Columns to Display:",
+        ),
+        search_columns=["Method"],
+        hide_columns=[],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
+def init_leaderboard(dataframe, track):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    # filter for correct track
+    dataframe = dataframe.loc[dataframe["Track"] == track]
+
+    # print(f"\n\n\n dataframe is {dataframe}\n\n\n")
+    
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+def process_json(temp_file):
+    if temp_file is None:
+        return {}
+
+    # Handle file upload
+    try:
+        file_path = temp_file.name
+        if file_path.endswith('.gz'):
+            with gzip.open(file_path, 'rt') as f:
+                data = json.load(f)
+        else:
+            with open(file_path, 'r') as f:
+                data = json.load(f)
+    except Exception as e:
+        raise gr.Error(f"Error processing file: {str(e)}")
+
+    gr.Markdown("Upload successful!")
+    return data
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # with gr.TabItem("Strict", elem_id="strict-benchmark-tab-table", id=0):
+        #     leaderboard = init_leaderboard(LEADERBOARD_DF, "strict")
+        # with gr.TabItem("Strict-small", elem_id="strict-small-benchmark-tab-table", id=1):
+        #     leaderboard = init_leaderboard(LEADERBOARD_DF, "strict-small")
+        # with gr.TabItem("Multimodal", elem_id="multimodal-benchmark-tab-table", id=2):
+        #     leaderboard = init_leaderboard(LEADERBOARD_DF_MULTIMODAL, "multimodal")
+
+        # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=4):
+        #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+ 
+        # with gr.TabItem("👶 Submit", elem_id="llm-benchmark-tab-table", id=5):
+        #     with gr.Column():
+        #         with gr.Row():                                                                                                           
+        #             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+        
+        # with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
+        #     leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
+        with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
+            # Add description for filters
+            gr.Markdown("""
+            ### Filtering Options
+            Use the dropdown menus below to filter results by specific tasks or models. 
+            You can combine filters to see specific task-model combinations.
+            """)
+            leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
+            print(f"Leaderboard is {leaderboard}")
+    
+        # Then modify the Causal Graph tab section
+        with gr.TabItem("Causal Graph", elem_id="causalgraph", id=1):
+            with gr.Tabs() as causalgraph_tabs:
+                with gr.TabItem("Detailed View", id=0):
+                    leaderboard_detailed = init_leaderboard_mib_causalgraph(
+                        LEADERBOARD_DF_MIB_CAUSALGRAPH_DETAILED, 
+                        "Causal Graph"
+                    )
+                with gr.TabItem("Aggregated View", id=1):
+                    leaderboard_aggregated = init_leaderboard_mib_causalgraph(
+                        LEADERBOARD_DF_MIB_CAUSALGRAPH_AGGREGATED, 
+                        "Causal Graph"
+                    )
+                with gr.TabItem("Intervention Averaged", id=2):
+                    leaderboard_averaged = init_leaderboard_mib_causalgraph(
+                        LEADERBOARD_DF_MIB_CAUSALGRAPH_AVERAGED, 
+                        "Causal Graph"
+                    )
+        
+    # with gr.Row():
+    #     with gr.Accordion("📙 Citation", open=False):
+    #         citation_button = gr.Textbox(
+    #             value=CITATION_BUTTON_TEXT,
+    #             label=CITATION_BUTTON_LABEL,
+    #             lines=20,
+    #             elem_id="citation-button",
+    #             show_copy_button=True,
+    #         )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.launch(share=True, ssr_mode=False)

custom-select-columns.py

@@ -96,7 +96,7 @@ if __name__ == "__main__":
     
     # Define filters and mappings
     column_filters = {
-        "IOI Metrics": ["ioi"],
+        "IOI": ["ioi"],
         "Performance Metrics": ["performance"]
     }
     

eval-queue/.gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-queue/README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

eval-results-mib-causalgraph/.gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-results-mib-causalgraph/submissions/MCQA_results_Qwen_correct_choice_period_token.json

@@ -0,0 +1,1332 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Qwen2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6428571428571429
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6428571428571429
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8571428571428571
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8571428571428571
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8571428571428571
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6428571428571429
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7142857142857143
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7142857142857143
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8571428571428571
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8571428571428571
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_Qwen_last_correct_choice_token.json

@@ -0,0 +1,1332 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Qwen2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_Qwen_last_token.json

@@ -0,0 +1,1332 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Qwen2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9285714285714286
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.14285714285714285
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.35714285714285715
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6428571428571429
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6428571428571429
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7857142857142857
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.42857142857142855
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9285714285714286
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.07142857142857142
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.21428571428571427
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_Qwen_second_to_last_token.json

@@ -0,0 +1,1332 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Qwen2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_google_correct_choice_period_token.json

@@ -0,0 +1,1442 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Gemma2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.5
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.4
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.43333333333333335
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.43333333333333335
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.5
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.7333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.43333333333333335
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.6333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.23333333333333334
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.6333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.6333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.6333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.5666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.5666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.2
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.06666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.06666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.06666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_google_correct_choice_token.json

@@ -0,0 +1,1442 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Gemma2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_google_last_token.json

@@ -0,0 +1,1442 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Gemma2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.16666666666666666
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.23333333333333334
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.5666666666666667
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.23333333333333334
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.8
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.5
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.43333333333333335
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3333333333333333
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.5
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.26666666666666666
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.26666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.4666666666666667
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.26666666666666666
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.26666666666666666
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.23333333333333334
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9666666666666667
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9333333333333333
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13333333333333333
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_google_second_to_last_token.json

@@ -0,0 +1,1442 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "Gemma2ForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.03333333333333333
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_meta-llama_correct_choice_period_token.json

@@ -0,0 +1,1772 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "LlamaForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9347826086956522
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6521739130434783
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.8478260869565217
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6739130434782609
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.3695652173913043
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.8478260869565217
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6956521739130435
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6521739130434783
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.32608695652173914
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7608695652173914
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.7608695652173914
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.8260869565217391
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9565217391304348
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.8478260869565217
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.8260869565217391
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.15217391304347827
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.15217391304347827
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6739130434782609
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.6739130434782609
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.15217391304347827
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6956521739130435
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.5652173913043478
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.5869565217391305
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.10869565217391304
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.08695652173913043
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.08695652173913043
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.043478260869565216
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "26",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "27",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "28",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "29",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "30",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "31",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_meta-llama_correct_choice_token.json

@@ -0,0 +1,1772 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "LlamaForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "26",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "27",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "28",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "29",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "30",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "31",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_meta-llama_last_token.json

@@ -0,0 +1,1772 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "LlamaForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.15217391304347827
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2391304347826087
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2608695652173913
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.1956521739130435
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6086956521739131
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.391304347826087
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.17391304347826086
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2608695652173913
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6739130434782609
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.3695652173913043
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2391304347826087
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.30434782608695654
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6086956521739131
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.391304347826087
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2608695652173913
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.32608695652173914
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.5869565217391305
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.391304347826087
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.4782608695652174
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.5869565217391305
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.34782608695652173
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.2391304347826087
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.6956521739130435
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.717391304347826
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.2391304347826087
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.1956521739130435
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "26",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "27",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "28",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "29",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.9782608695652174
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "30",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "31",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.13043478260869565
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-causalgraph/submissions/MCQA_results_meta-llama_second_to_last_token.json

@@ -0,0 +1,1772 @@
+{
+  "method_name": "full_vector",
+  "results": [
+    {
+      "model_id": "LlamaForCausalLM",
+      "task_scores": {
+        "MCQA": [
+          {
+            "layer": "0",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "1",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "2",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "3",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "4",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "5",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "6",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "7",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "8",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "9",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "10",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "11",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "12",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "13",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "14",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "15",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "16",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "17",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "18",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "19",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "20",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "21",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "22",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "23",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "24",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "25",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "26",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "27",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "28",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "29",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "30",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "layer": "31",
+            "layer_scores": [
+              {
+                "intervention": [
+                  "output_token"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 0.021739130434782608
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.06521739130434782
+                  }
+                ]
+              },
+              {
+                "intervention": [
+                  "output_location"
+                ],
+                "counterfactual_scores": [
+                  {
+                    "counterfactual": [
+                      "randomLetter_counterfactual"
+                    ],
+                    "score": 1.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_counterfactual"
+                    ],
+                    "score": 0.0
+                  },
+                  {
+                    "counterfactual": [
+                      "answerPosition_randomLetter_counterfactual"
+                    ],
+                    "score": 0.0
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

eval-results-mib-subgraph/.gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mds filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-results-mib-subgraph/README.md

@@ -0,0 +1,3 @@
+---
+license: apache-2.0
+---

eval-results-mib-subgraph/baselines/EAP-IG-activations_CF.json

@@ -0,0 +1,337 @@
+{
+    "method_name": "EAP-IG-activations (CF)",
+    "results": [
+        {
+            "model_id": "llama3",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        1147.0,
+                        2643.0,
+                        7461.0,
+                        15315.0,
+                        31091.0,
+                        78709.0,
+                        157644.0,
+                        314446.0,
+                        779049.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.03076923076923077,
+                        0.16923076923076924,
+                        0.5303846153846153,
+                        0.8692307692307693,
+                        0.9415384615384615,
+                        1.0307692307692307,
+                        1.0769230769230769,
+                        1.083076923076923,
+                        1.0923076923076922,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        1150.0,
+                        2654.0,
+                        7471.0,
+                        15345.0,
+                        31203.0,
+                        78828.0,
+                        158056.0,
+                        315717.0,
+                        777259.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0019305019305019305,
+                        0.005791505791505791,
+                        0.019305019305019305,
+                        0.04247104247104247,
+                        0.03088803088803089,
+                        -0.06563706563706563,
+                        -0.09652509652509653,
+                        -0.12355212355212356,
+                        -0.15057915057915058,
+                        1.0
+                    ]
+                },
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        1193.0,
+                        2659.0,
+                        7427.0,
+                        15403.0,
+                        31137.0,
+                        78007.0,
+                        156372.0,
+                        311572.0,
+                        767443.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.002352941176470588,
+                        0.002352941176470588,
+                        0.018823529411764704,
+                        0.07529411764705882,
+                        0.17647058823529413,
+                        0.30823529411764705,
+                        0.32,
+                        0.30941176470588233,
+                        0.31058823529411766,
+                        1.0
+                    ]
+                },
+                "arc_challenge": {
+                    "edge_counts": [
+                        912.0,
+                        2002.0,
+                        6665.0,
+                        14879.0,
+                        30445.0,
+                        77949.0,
+                        156864.0,
+                        312741.0,
+                        748529.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0024096385542168677,
+                        0.03132530120481928,
+                        1.0
+                    ]
+                },
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        1257.0,
+                        2773.0,
+                        7536.0,
+                        15449.0,
+                        31202.0,
+                        78608.0,
+                        157421.0,
+                        313702.0,
+                        773168.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.8605769230769231,
+                        0.9326923076923077,
+                        1.0192307692307692,
+                        1.0032051282051282,
+                        0.9967948717948718,
+                        0.9551282051282052,
+                        0.9487179487179487,
+                        0.9551282051282052,
+                        0.9583333333333334,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        757.0,
+                        1896.0,
+                        6791.0,
+                        14846.0,
+                        30465.0,
+                        78485.0,
+                        156768.0,
+                        313872.0,
+                        747107.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0023584905660377358,
+                        0.0023584905660377358,
+                        0.0023584905660377358,
+                        0.0023584905660377358,
+                        0.0023584905660377358,
+                        -0.01179245283018868,
+                        -0.009433962264150943,
+                        0.007075471698113208,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        29.0,
+                        58.0,
+                        127.0,
+                        232.0,
+                        744.0,
+                        2697.0,
+                        6288.0,
+                        13926.0,
+                        37109.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.022508038585209004,
+                        0.364951768488746,
+                        0.5610932475884244,
+                        0.6302250803858521,
+                        0.657556270096463,
+                        0.6366559485530546,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        46.0,
+                        88.0,
+                        152.0,
+                        319.0,
+                        885.0,
+                        3049.0,
+                        6845.0,
+                        14362.0,
+                        36021.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.025510204081632654,
+                        0.06292517006802721,
+                        0.07993197278911565,
+                        0.17687074829931973,
+                        0.32950680272108845,
+                        0.8775510204081632,
+                        1.251700680272109,
+                        1.4829931972789117,
+                        1.6258503401360545,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        44.0,
+                        78.0,
+                        131.0,
+                        260.0,
+                        596.0,
+                        2290.0,
+                        5683.0,
+                        13168.0,
+                        36989.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        -0.0017730496453900709,
+                        0.0,
+                        0.0,
+                        -0.0035460992907801418,
+                        0.07801418439716312,
+                        0.14361702127659576,
+                        0.18085106382978725,
+                        0.2666223404255319,
+                        0.12677304964539007,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        1.0,
+                        5.0,
+                        31.0,
+                        116.0,
+                        302.0,
+                        1211.0,
+                        2824.0,
+                        5987.0,
+                        14986.0,
+                        32491.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.00020081878619851592,
+                        -0.0020080188226028373,
+                        -0.005186060633885513,
+                        -0.006654032437846079,
+                        0.0611483737769071,
+                        0.15180992495124734,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        142.0,
+                        267.0,
+                        724.0,
+                        1612.0,
+                        3432.0,
+                        8817.0,
+                        17776.0,
+                        35593.0,
+                        87039.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.317246835443038,
+                        0.45450949367088606,
+                        0.5593354430379747,
+                        0.5229430379746836,
+                        0.5704113924050633,
+                        0.5514240506329114,
+                        0.7088607594936709,
+                        0.6930379746835443,
+                        0.7341772151898734,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        133.0,
+                        285.0,
+                        748.0,
+                        1634.0,
+                        3321.0,
+                        8607.0,
+                        17300.0,
+                        34763.0,
+                        87704.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.44820872274143303,
+                        0.6884735202492211,
+                        1.1090342679127725,
+                        1.3707165109034267,
+                        1.557632398753894,
+                        1.7757009345794392,
+                        1.8193146417445483,
+                        1.8255451713395638,
+                        1.8130841121495327,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/EAP-IG-inputs_CF.json

@@ -0,0 +1,311 @@
+{
+    "method_name": "EAP-IG-inputs (CF)",
+    "results": [
+        {
+            "model_id": "llama3",
+            "scores": {
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        1197.0,
+                        2573.0,
+                        7447.0,
+                        15416.0,
+                        31203.0,
+                        78484.0,
+                        156784.0,
+                        312065.0,
+                        769311.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        1.0141176470588236,
+                        1.051764705882353,
+                        1.0635294117647058,
+                        1.0776470588235294,
+                        1.0258823529411765,
+                        0.9623529411764706,
+                        0.9058823529411765,
+                        0.8776470588235294,
+                        0.8564705882352941,
+                        1.0
+                    ]
+                },
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        1193.0,
+                        2592.0,
+                        7327.0,
+                        15198.0,
+                        30956.0,
+                        78020.0,
+                        156182.0,
+                        311361.0,
+                        768765.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        1.1570512820512822,
+                        1.1923076923076923,
+                        1.2211538461538463,
+                        1.189102564102564,
+                        1.1346153846153846,
+                        1.1057692307692308,
+                        1.0769230769230769,
+                        1.0320512820512822,
+                        1.0256410256410255,
+                        1.0
+                    ]
+                },
+                "arc_challenge": {
+                    "edge_counts": [
+                        1072.0,
+                        2186.0,
+                        6683.0,
+                        14838.0,
+                        30868.0,
+                        78580.0,
+                        157911.0,
+                        313964.0,
+                        783154.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0024390243902439024,
+                        0.004878048780487805,
+                        0.06341463414634146,
+                        0.07073170731707316,
+                        0.05853658536585366,
+                        0.07926829268292683,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        1239.0,
+                        2680.0,
+                        7481.0,
+                        15381.0,
+                        31252.0,
+                        78706.0,
+                        157634.0,
+                        314876.0,
+                        777087.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.9208333333333333,
+                        0.9916666666666667,
+                        1.2041666666666666,
+                        1.1833333333333333,
+                        1.0625,
+                        1.0,
+                        1.075,
+                        1.05,
+                        1.05,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        1248.0,
+                        2764.0,
+                        7492.0,
+                        15370.0,
+                        31235.0,
+                        78615.0,
+                        157388.0,
+                        314400.0,
+                        777958.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.6019076305220884,
+                        0.8032128514056225,
+                        1.0461847389558232,
+                        1.0983935742971886,
+                        1.0140562248995983,
+                        0.8152610441767069,
+                        0.8815261044176707,
+                        1.0301204819277108,
+                        1.2429718875502007,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        124.0,
+                        287.0,
+                        757.0,
+                        1603.0,
+                        3306.0,
+                        8490.0,
+                        17268.0,
+                        34572.0,
+                        87156.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.5985202492211839,
+                        0.9314641744548287,
+                        1.6635514018691588,
+                        1.925233644859813,
+                        2.155763239875389,
+                        2.2554517133956384,
+                        2.2679127725856696,
+                        2.2803738317757007,
+                        2.2679127725856696,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        131.0,
+                        267.0,
+                        751.0,
+                        1592.0,
+                        3412.0,
+                        8807.0,
+                        17709.0,
+                        35540.0,
+                        87132.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.42764195583596215,
+                        0.7066246056782335,
+                        0.8517350157728707,
+                        1.0094637223974763,
+                        1.2176656151419558,
+                        1.337539432176656,
+                        1.2618296529968454,
+                        1.1230283911671923,
+                        1.0725552050473186,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        41.0,
+                        84.0,
+                        173.0,
+                        381.0,
+                        1003.0,
+                        3195.0,
+                        6914.0,
+                        14394.0,
+                        35995.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.10374149659863946,
+                        0.27253401360544216,
+                        0.4574829931972789,
+                        0.9217687074829932,
+                        1.3129251700680271,
+                        1.564625850340136,
+                        1.6734693877551021,
+                        1.91156462585034,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        28.0,
+                        77.0,
+                        269.0,
+                        624.0,
+                        1342.0,
+                        3526.0,
+                        7322.0,
+                        14745.0,
+                        36995.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.04180064308681672,
+                        0.36897106109324757,
+                        1.77491961414791,
+                        2.707395498392283,
+                        3.247588424437299,
+                        4.147909967845659,
+                        4.508038585209003,
+                        4.713826366559486,
+                        4.816720257234727,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        38.0,
+                        107.0,
+                        228.0,
+                        443.0,
+                        988.0,
+                        3232.0,
+                        7086.0,
+                        14674.0,
+                        36641.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0736196319018405,
+                        0.18404907975460122,
+                        0.4302147239263804,
+                        0.9447852760736196,
+                        1.1748466257668713,
+                        1.549079754601227,
+                        1.6349693251533743,
+                        1.7576687116564418,
+                        1.9171779141104295,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        28.17708396911621,
+                        74.19965362548828,
+                        317.2265625,
+                        881.1848754882812,
+                        2312.229248046875,
+                        7085.25,
+                        15016.50390625,
+                        30456.0,
+                        76492.484375,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.11661430390775049,
+                        0.4642177695188903,
+                        1.4695986430872536,
+                        1.8416996434283752,
+                        2.103407867294637,
+                        2.3131087656712164,
+                        2.385959455571554,
+                        2.406932236515815,
+                        2.365694611464723,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/EAP_CF.json

@@ -0,0 +1,285 @@
+{
+    "method_name": "EAP (CF)",
+    "results": [
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        40.0,
+                        76.0,
+                        176.0,
+                        332.0,
+                        962.0,
+                        3256.0,
+                        6866.0,
+                        14441.0,
+                        36027.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.05782312925170068,
+                        0.06462585034013606,
+                        0.13435374149659865,
+                        0.19727891156462585,
+                        0.3521471088435374,
+                        0.9591836734693877,
+                        1.2857142857142858,
+                        1.435374149659864,
+                        1.5510204081632653,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        33.0,
+                        85.0,
+                        231.0,
+                        451.0,
+                        1054.0,
+                        3305.0,
+                        7024.0,
+                        14506.0,
+                        36031.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.04447852760736196,
+                        0.06441717791411043,
+                        0.21779141104294478,
+                        0.3159509202453988,
+                        0.5076687116564417,
+                        0.8374233128834356,
+                        1.0245398773006136,
+                        1.138036809815951,
+                        1.334355828220859,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        1193.0,
+                        2592.0,
+                        7327.0,
+                        15198.0,
+                        30956.0,
+                        78020.0,
+                        156182.0,
+                        311361.0,
+                        768765.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.08709677419354839,
+                        0.16612903225806452,
+                        0.37661290322580643,
+                        0.5518145161290322,
+                        0.8741935483870967,
+                        1.0354838709677419,
+                        1.3612903225806452,
+                        1.3612903225806452,
+                        1.4645161290322581,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        110.0,
+                        246.0,
+                        653.0,
+                        1455.0,
+                        3114.0,
+                        8393.0,
+                        17273.0,
+                        34695.0,
+                        87690.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.01557632398753894,
+                        0.021806853582554516,
+                        0.018691588785046728,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        128.0,
+                        262.0,
+                        724.0,
+                        1586.0,
+                        3413.0,
+                        8813.0,
+                        17768.0,
+                        35535.0,
+                        86556.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.27681388012618297,
+                        0.3994479495268139,
+                        0.5441640378548895,
+                        0.5291798107255521,
+                        0.48738170347003157,
+                        0.5977917981072555,
+                        0.7523659305993691,
+                        0.6876971608832808,
+                        0.667192429022082,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        28.17708396911621,
+                        74.19965362548828,
+                        317.2265625,
+                        881.1848754882812,
+                        2312.229248046875,
+                        7085.25,
+                        15016.50390625,
+                        30456.0,
+                        76492.484375,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.22449694731820746,
+                        0.5048270710448762,
+                        0.7968936313069007,
+                        0.976166532167683,
+                        1.2088630716953397,
+                        1.232255211567972,
+                        1.2651923955277595,
+                        1.3237581777016123,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "llama3",
+            "scores": {
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        1101.0,
+                        2521.0,
+                        7330.0,
+                        15268.0,
+                        31128.0,
+                        78568.0,
+                        157165.0,
+                        313324.0,
+                        773309.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.21634615384615385,
+                        0.3076923076923077,
+                        0.4110576923076923,
+                        0.4467147435897436,
+                        0.49854767628205127,
+                        0.5422676282051282,
+                        0.5584935897435898,
+                        0.5737179487179487,
+                        0.577323717948718,
+                        1.0
+                    ]
+                },
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        1105.0,
+                        2636.0,
+                        7488.0,
+                        15424.0,
+                        31195.0,
+                        78423.0,
+                        156792.0,
+                        312466.0,
+                        776673.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.08,
+                        0.13411764705882354,
+                        0.18352941176470589,
+                        0.22588235294117648,
+                        0.2847058823529412,
+                        0.3588235294117647,
+                        0.37529411764705883,
+                        0.3841176470588235,
+                        0.3988235294117647,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        1204.0,
+                        2743.0,
+                        7462.0,
+                        15300.0,
+                        31172.0,
+                        78615.0,
+                        157496.0,
+                        314048.0,
+                        767228.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.014056224899598393,
+                        0.04618473895582329,
+                        0.1144578313253012,
+                        0.19678714859437751,
+                        0.19076305220883535,
+                        0.3644578313253012,
+                        0.5144327309236948,
+                        0.7590361445783133,
+                        0.7570281124497992,
+                        1.0
+                    ]
+                },
+                "arc_challenge": {
+                    "edge_counts": [
+                        1080.0,
+                        2633.0,
+                        6870.0,
+                        14850.0,
+                        30338.0,
+                        77983.0,
+                        156741.0,
+                        306677.0,
+                        781379.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        -0.0024390243902439024,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.014634146341463415,
+                        0.007317073170731708,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/EAP_OA.json

@@ -0,0 +1,93 @@
+{
+    "method_name": "EAP (OA)",
+    "results": [
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        3.0,
+                        8.0,
+                        18.0,
+                        66.0,
+                        119.0,
+                        616.0,
+                        2699.0,
+                        6055.0,
+                        14988.0,
+                        32491.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        -0.02605376953427554,
+                        -0.027599262798968967,
+                        0.018682773460249002,
+                        0.3035174289407458,
+                        0.905198792680162,
+                        1.2374291327745524,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        19.0,
+                        81.0,
+                        203.0,
+                        360.0,
+                        872.0,
+                        7146.0,
+                        17450.0,
+                        35719.0,
+                        87921.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        -0.0031645569620253164,
+                        -0.006329113924050633,
+                        -0.015822784810126583,
+                        0.03164556962025317,
+                        0.09651898734177215,
+                        0.189873417721519,
+                        0.13132911392405064,
+                        0.07278481012658228,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        7.0,
+                        62.0,
+                        163.0,
+                        393.0,
+                        868.0,
+                        7103.0,
+                        17098.0,
+                        35450.0,
+                        88180.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.16043613707165108,
+                        0.22429906542056074,
+                        0.38317757009345793,
+                        0.5595794392523364,
+                        0.6433021806853583,
+                        0.6448598130841121,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/EAP_mean.json

@@ -0,0 +1,311 @@
+{
+    "method_name": "EAP (mean)",
+    "results": [
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        31.0,
+                        71.0,
+                        139.0,
+                        266.0,
+                        746.0,
+                        2672.0,
+                        6280.0,
+                        13367.0,
+                        36357.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        -0.003401360544217687,
+                        -0.01020408163265306,
+                        0.07312925170068027,
+                        0.06292517006802721,
+                        -0.006802721088435374,
+                        -0.1326530612244898,
+                        -0.22108843537414966,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        32.0,
+                        66.0,
+                        154.0,
+                        266.0,
+                        717.0,
+                        2773.0,
+                        6332.0,
+                        13490.0,
+                        36992.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        -0.0015337423312883436,
+                        0.032208588957055216,
+                        0.05061349693251534,
+                        0.08742331288343558,
+                        0.0705521472392638,
+                        0.1303680981595092,
+                        0.09202453987730061,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        14.0,
+                        28.0,
+                        103.0,
+                        296.0,
+                        866.0,
+                        3074.0,
+                        6433.0,
+                        13536.0,
+                        36793.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.02572347266881029,
+                        0.26688102893890675,
+                        0.2990353697749196,
+                        0.39147909967845657,
+                        0.5315012057877814,
+                        0.5836012861736335,
+                        0.6213826366559485,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "llama3",
+            "scores": {
+                "arc_easy": {
+                    "edge_counts": [
+                        1062.0,
+                        2532.0,
+                        7119.0,
+                        14925.0,
+                        30758.0,
+                        78089.0,
+                        156448.0,
+                        313277.0,
+                        777481.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0020833333333333333,
+                        0.035416666666666666,
+                        0.075,
+                        0.115625,
+                        0.13854166666666667,
+                        0.13958333333333334,
+                        0.12291666666666666,
+                        1.0
+                    ]
+                },
+                "arc_challenge": {
+                    "edge_counts": [
+                        1040.0,
+                        2529.0,
+                        6078.0,
+                        14242.0,
+                        29098.0,
+                        74505.0,
+                        152010.0,
+                        306198.0,
+                        778836.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        -0.0024390243902439024,
+                        -0.0024390243902439024,
+                        0.0,
+                        0.004878048780487805,
+                        0.05365853658536585,
+                        0.02926829268292683,
+                        0.02195121951219512,
+                        0.03170731707317073,
+                        0.1048780487804878,
+                        1.0
+                    ]
+                },
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        1125.0,
+                        2578.0,
+                        7432.0,
+                        15224.0,
+                        30876.0,
+                        77771.0,
+                        155258.0,
+                        309527.0,
+                        774834.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.003205128205128205,
+                        0.016025641025641024,
+                        0.028846153846153848,
+                        0.041666666666666664,
+                        0.10256410256410256,
+                        0.10096153846153846,
+                        0.11538461538461539,
+                        0.11538461538461539,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        1044.0,
+                        2503.0,
+                        7278.0,
+                        15027.0,
+                        30930.0,
+                        78647.0,
+                        157997.0,
+                        315572.0,
+                        777353.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.004016064257028112,
+                        0.008032128514056224,
+                        0.012048192771084338,
+                        0.050200803212851405,
+                        0.13253012048192772,
+                        0.09437751004016064,
+                        0.10843373493975904,
+                        0.11646586345381527,
+                        1.0
+                    ]
+                },
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        1148.0,
+                        2687.0,
+                        7431.0,
+                        15219.0,
+                        30812.0,
+                        77526.0,
+                        154898.0,
+                        307910.0,
+                        769045.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.002352941176470588,
+                        0.018823529411764704,
+                        0.07058823529411765,
+                        0.18352941176470589,
+                        0.27647058823529413,
+                        0.31411764705882356,
+                        0.31882352941176473,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        28.17708396911621,
+                        74.19965362548828,
+                        317.2265625,
+                        881.1848754882812,
+                        2312.229248046875,
+                        7085.25,
+                        15016.50390625,
+                        30456.0,
+                        76492.484375,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        -0.00020504474302805614,
+                        -0.0014366993239439858,
+                        0.0008672395297865946,
+                        -0.012577542580169451,
+                        0.08428255131781394,
+                        0.05067079889015518,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        58.0,
+                        119.0,
+                        434.0,
+                        1327.0,
+                        3168.0,
+                        8434.0,
+                        17304.0,
+                        35310.0,
+                        87249.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.006329113924050633,
+                        0.006329113924050633,
+                        -0.0189873417721519,
+                        -0.022151898734177215,
+                        0.006329113924050633,
+                        0.0,
+                        0.1439873417721519,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        26.0,
+                        85.0,
+                        298.0,
+                        842.0,
+                        2512.0,
+                        7868.0,
+                        16655.0,
+                        34761.0,
+                        86664.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.5311039719626168,
+                        0.5891744548286605,
+                        0.6456386292834891,
+                        0.6401869158878505,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/IFR.json

@@ -0,0 +1,285 @@
+{
+    "method_name": "IFR",
+    "results": [
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        60.0,
+                        697.0,
+                        2012.0,
+                        27442.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        34.0,
+                        569.0,
+                        1843.0,
+                        23583.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        48.0,
+                        282.0,
+                        1912.0,
+                        36317.0,
+                        74218.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.18877551020408162,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "llama3",
+            "scores": {
+                "arc_easy": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        81121.0,
+                        678541.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.24895833333333334,
+                        1.0
+                    ]
+                },
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        19009.0,
+                        245201.0,
+                        728253.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.9435294117647058,
+                        1.3129411764705883,
+                        1.0
+                    ]
+                },
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        1719.0,
+                        104811.0,
+                        210163.0,
+                        726283.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.39503205128205127,
+                        0.9198717948717948,
+                        1.2788461538461537,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        11420.0,
+                        249100.0,
+                        687383.0,
+                        1592881.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.07228915662650602,
+                        0.7018072289156626,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        1.0,
+                        5.0,
+                        18.0,
+                        61.0,
+                        431.0,
+                        2217.0,
+                        11133.0,
+                        32491.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        9.699480413473062e-05,
+                        0.002604457555290081,
+                        0.005239274585697969,
+                        0.007735479313229455,
+                        0.8286768164229479,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        6114.0,
+                        78382.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.6593059936908517,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        18.0,
+                        18.0,
+                        8519.0,
+                        77692.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.20249221183800623,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/NAP-IG-inputs_CF.json

@@ -0,0 +1,337 @@
+{
+    "method_name": "NAP-IG-inputs (CF)",
+    "results": [
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        109.0,
+                        256.0,
+                        673.0,
+                        897.0,
+                        1203.0,
+                        2237.0,
+                        6224.0,
+                        16053.0,
+                        57617.0,
+                        179749.0
+                    ],
+                    "faithfulness": [
+                        0.47001557632398755,
+                        0.6954828660436138,
+                        1.087227414330218,
+                        1.087227414330218,
+                        1.1059190031152648,
+                        1.1246105919003115,
+                        1.2024922118380061,
+                        1.2118380062305296,
+                        1.3582554517133956,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        15.861607551574707,
+                        39.65401840209961,
+                        223.78125,
+                        978.140625,
+                        3732.670654296875,
+                        15233.4462890625,
+                        34149.96484375,
+                        71549.9765625,
+                        185915.046875,
+                        372913.0
+                    ],
+                    "faithfulness": [
+                        -0.0031645569620253164,
+                        0.0,
+                        0.0,
+                        0.012658227848101266,
+                        0.0379746835443038,
+                        0.11708860759493671,
+                        0.2120253164556962,
+                        0.27610759493670883,
+                        0.49802215189873417,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "mcqa": {
+                    "edge_counts": [
+                        20.42838478088379,
+                        76.078125,
+                        334.8502502441406,
+                        1029.2239990234375,
+                        2481.416748046875,
+                        7191.0,
+                        15016.498046875,
+                        30315.009765625,
+                        76492.484375,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.006802721088435374,
+                        0.01020408163265306,
+                        0.03571428571428571,
+                        0.07482993197278912,
+                        0.16071428571428573,
+                        0.5068027210884354,
+                        1.119047619047619,
+                        2.17687074829932,
+                        3.061224489795918,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        20.19357681274414,
+                        76.078125,
+                        296.078125,
+                        852.9869995117188,
+                        2340.427001953125,
+                        7191.0,
+                        14946.0,
+                        30456.0,
+                        76492.5078125,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.009202453987730062,
+                        0.01687116564417178,
+                        0.04141104294478527,
+                        0.08282208588957055,
+                        0.18251533742331288,
+                        0.4854294478527607,
+                        1.0582822085889572,
+                        1.7699386503067485,
+                        2.303680981595092,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        5.87022590637207,
+                        28.17708396911621,
+                        155.0885467529297,
+                        505.2126770019531,
+                        1362.8992919921875,
+                        5111.25,
+                        13254.0,
+                        29892.0,
+                        76492.515625,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.006430868167202572,
+                        0.04823151125401929,
+                        0.2877813504823151,
+                        0.882636655948553,
+                        1.6463022508038585,
+                        2.1607717041800645,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        1.25,
+                        4.70703125,
+                        23.515625,
+                        67.46744537353516,
+                        197.77734375,
+                        1279.3802490234375,
+                        3971.90234375,
+                        13030.8916015625,
+                        34932.5,
+                        69865.0
+                    ],
+                    "faithfulness": [
+                        -9.80428417467502e-07,
+                        7.268693439845274e-06,
+                        -3.6850585346192315e-05,
+                        0.00037178521747896964,
+                        0.005384073366337484,
+                        0.05083179885016707,
+                        0.17342828703671462,
+                        0.4600666180824943,
+                        1.0150521456067374,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "llama3",
+            "scores": {
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        66.583740234375,
+                        238.85595703125,
+                        2013.551513671875,
+                        8403.009765625,
+                        32026.939453125,
+                        120814.765625,
+                        276828.1875,
+                        615173.8125,
+                        1639407.0,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        -0.002352941176470588,
+                        0.0,
+                        0.002352941176470588,
+                        0.01647058823529412,
+                        0.08,
+                        0.1811764705882353,
+                        0.2858823529411765,
+                        0.36764705882352944,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        453.403564453125,
+                        1490.20751953125,
+                        6706.5537109375,
+                        19596.455078125,
+                        50291.8125,
+                        148085.3125,
+                        313294.65625,
+                        649420.5,
+                        1639406.875,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0019305019305019305,
+                        0.0019305019305019305,
+                        0.007722007722007722,
+                        0.025096525096525095,
+                        0.11196911196911197,
+                        0.9884169884169884,
+                        1.9652509652509653,
+                        2.305019305019305,
+                        2.4208494208494207,
+                        1.0
+                    ]
+                },
+                "arc_challenge": {
+                    "edge_counts": [
+                        24.308349609375,
+                        107.80224609375,
+                        317.0947265625,
+                        697.6083984375,
+                        7103.00390625,
+                        77689.2890625,
+                        278096.5625,
+                        649420.5,
+                        1640992.625,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        -0.0024390243902439024,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.004878048780487805,
+                        0.012195121951219513,
+                        1.0
+                    ]
+                },
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        53.901123046875,
+                        257.8798828125,
+                        1839.1494140625,
+                        9068.91015625,
+                        32851.39453125,
+                        120022.0390625,
+                        273340.0625,
+                        613271.0625,
+                        1639407.0,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.003205128205128205,
+                        0.0,
+                        0.00641025641025641,
+                        0.022435897435897436,
+                        0.08012820512820513,
+                        0.17467948717948717,
+                        0.2644230769230769,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        211.376953125,
+                        629.9033203125,
+                        2991.26025390625,
+                        9248.595703125,
+                        27418.44140625,
+                        109716.296875,
+                        271754.59375,
+                        614539.625,
+                        1640992.5,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.003076923076923077,
+                        0.015384615384615385,
+                        0.07384615384615385,
+                        0.14923076923076922,
+                        0.2323076923076923,
+                        0.2846153846153846,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        17.967041015625,
+                        93.005859375,
+                        301.239990234375,
+                        697.6083984375,
+                        8751.9150390625,
+                        80226.0859375,
+                        281901.75,
+                        648152.1875,
+                        1640992.375,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.005,
+                        0.015,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/baselines/NAP_CF.json

@@ -0,0 +1,337 @@
+{
+    "method_name": "NAP (CF)",
+    "results": [
+        {
+            "model_id": "gemma2",
+            "scores": {
+                "arc_easy": {
+                    "edge_counts": [
+                        28.17708396911621,
+                        74.19965362548828,
+                        317.2265625,
+                        881.1848754882812,
+                        2312.229248046875,
+                        7085.25,
+                        15016.50390625,
+                        30456.0,
+                        76492.484375,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.004601226993865031,
+                        0.006134969325153374,
+                        0.013803680981595092,
+                        0.02607361963190184,
+                        0.04754601226993865,
+                        0.12423312883435583,
+                        0.24233128834355827,
+                        0.5751533742331288,
+                        1.6411042944785277,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        44.14409637451172,
+                        142.76388549804688,
+                        553.3840942382812,
+                        1318.2525634765625,
+                        2904.385498046875,
+                        7578.75,
+                        15298.49609375,
+                        30597.0078125,
+                        76492.4765625,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        -0.003401360544217687,
+                        -0.003401360544217687,
+                        -0.0017006802721088435,
+                        0.0017006802721088435,
+                        0.02040816326530612,
+                        0.022108843537414966,
+                        0.10204081632653061,
+                        0.24489795918367346,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        78.42621612548828,
+                        195.36111450195312,
+                        587.4566040039062,
+                        1273.60595703125,
+                        2791.59375,
+                        7614.0,
+                        15298.5,
+                        30596.99609375,
+                        76492.484375,
+                        152985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.003215434083601286,
+                        0.01929260450160772,
+                        0.07395498392282958,
+                        0.1607717041800643,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "llama3",
+            "scores": {
+                "arithmetic_subtraction": {
+                    "edge_counts": [
+                        210.320068359375,
+                        596.0830078125,
+                        3139.23779296875,
+                        11288.572265625,
+                        34563.7265625,
+                        117960.8671875,
+                        271754.5625,
+                        614539.5,
+                        1639406.875,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.003205128205128205,
+                        0.00641025641025641,
+                        0.01282051282051282,
+                        0.041666666666666664,
+                        0.08012820512820513,
+                        1.0
+                    ]
+                },
+                "arc_easy": {
+                    "edge_counts": [
+                        22.194580078125,
+                        59.185546875,
+                        248.390869140625,
+                        507.3515625,
+                        9259.2734375,
+                        77213.6328125,
+                        270803.25,
+                        646249.5625,
+                        1640992.125,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.02,
+                        1.0
+                    ]
+                },
+                "arc_challenge": {
+                    "edge_counts": [
+                        31.70654296875,
+                        90.89208984375,
+                        232.5361328125,
+                        792.73681640625,
+                        9893.4697265625,
+                        64371.125,
+                        143329.125,
+                        628492.0,
+                        1640992.375,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.012195121951219513,
+                        1.0
+                    ]
+                },
+                "ioi": {
+                    "edge_counts": [
+                        361.45458984375,
+                        1048.4296875,
+                        4175.08056640625,
+                        11753.64453125,
+                        31984.65625,
+                        109082.09375,
+                        260973.203125,
+                        612637.0,
+                        1640992.375,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.009230769230769232,
+                        0.015384615384615385,
+                        0.015384615384615385,
+                        1.0
+                    ]
+                },
+                "arithmetic_addition": {
+                    "edge_counts": [
+                        161.703369140625,
+                        450.23291015625,
+                        2742.869384765625,
+                        10559.25390625,
+                        34246.625,
+                        117802.3125,
+                        275559.8125,
+                        616442.0625,
+                        1639407.0,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        -0.002352941176470588,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.004705882352941176,
+                        0.011764705882352941,
+                        0.0,
+                        0.0,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        700.714599609375,
+                        2084.1767578125,
+                        9259.166015625,
+                        23591.84765625,
+                        54921.44140625,
+                        153634.5,
+                        320587.96875,
+                        655762.5625,
+                        1640992.375,
+                        3281985.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0019305019305019305,
+                        0.0019305019305019305,
+                        0.0,
+                        -0.007722007722007722,
+                        -0.02702702702702703,
+                        -0.07722007722007722,
+                        -0.05791505791505792,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "qwen2.5",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        32.8046875,
+                        88.68080139160156,
+                        348.3046875,
+                        1010.625,
+                        3039.563720703125,
+                        11605.576171875,
+                        29601.451171875,
+                        70466.984375,
+                        185914.984375,
+                        372913.0
+                    ],
+                    "faithfulness": [
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.0,
+                        0.006230529595015576,
+                        0.024922118380062305,
+                        0.04984423676012461,
+                        0.06853582554517133,
+                        0.08099688473520249,
+                        1.0
+                    ]
+                },
+                "mcqa": {
+                    "edge_counts": [
+                        21.98995590209961,
+                        100.21651458740234,
+                        516.140625,
+                        1844.390625,
+                        5357.140625,
+                        17507.634765625,
+                        37182.30859375,
+                        74582.3984375,
+                        186456.53125,
+                        372913.0
+                    ],
+                    "faithfulness": [
+                        -0.0031645569620253164,
+                        -0.0031645569620253164,
+                        -0.0031645569620253164,
+                        -0.0031645569620253164,
+                        -0.0031645569620253164,
+                        0.012658227848101266,
+                        0.012658227848101266,
+                        0.03481012658227848,
+                        0.056962025316455694,
+                        1.0
+                    ]
+                }
+            }
+        },
+        {
+            "model_id": "gpt2",
+            "scores": {
+                "ioi": {
+                    "edge_counts": [
+                        3.28125,
+                        10.35546875,
+                        31.35416603088379,
+                        83.15755462646484,
+                        279.3997497558594,
+                        1350.0208740234375,
+                        4442.87890625,
+                        12654.0947265625,
+                        34932.49609375,
+                        69865.0
+                    ],
+                    "faithfulness": [
+                        -3.380787646439662e-08,
+                        0.0,
+                        -1.5450199544229257e-05,
+                        -6.119225640055789e-06,
+                        0.00011721190770206309,
+                        0.0020236718694058528,
+                        0.015380555318712599,
+                        0.03909438029925786,
+                        0.04813088759942643,
+                        1.0
+                    ]
+                }
+            }
+        }
+    ]
+}

eval-results-mib-subgraph/submissions/results_2024-10-2T13-36-121.json

@@ -0,0 +1,19 @@
+{"method_name": "EAP-IG (mean)", "results": [
+    {"model_id": "meta-llama/Llama-3.1-8B", "scores": {
+        "ioi": {
+            "edge_counts": [10.0, 29.0, 117.0, 269.0, 561.0, 1570.0, 3194.0, 6386.0, 16245.0, 32491.0],
+            "faithfulness": [0.11454112510535433,0.14123527363014815,0.3197643850972241,0.47765884872924175,0.7701570853704176,1.3201798748760563,2.037825774185549,2.651813181821849,3.27612042118584,1.0]},
+        "mcqa": {
+            "edge_counts": [10.0, 21.0, 94.0, 241.0, 527.0, 1469.0, 3046.0, 6036.0, 14832.0, 32491.0],
+            "faithfulness": [[0.02677059664121319,0.1965060952906922,0.449060470868564,0.7604756153676078,0.786575587658478,1.106011020720112,1.3436645156597262,1.5466349080478032,1.4914126224418107,1.0]]}
+    }},
+    {"model_id": "Qwen/Qwen2-1.5B", "scores": {
+        "ioi": {
+        "edge_counts": [],
+         "faithfulness": []},
+        "mcqa": {
+        "edge_counts": [],
+        "faithfulness": []}
+    }}
+    ]
+}

src/leaderboard/read_evals.py

@@ -99,7 +99,7 @@ class EvalResult_MIB_SUBGRAPH:
 
 
     
-    def to_dict(self):
+    def to_dict(self, metric_type="F+"):
         """Converts the Eval Result to a dict for dataframe display"""
         data_dict = {
             "eval_name": self.eval_name,
@@ -140,8 +140,8 @@ class EvalResult_MIB_SUBGRAPH:
                 if result is None or result[0] is None:
                     continue
                     
-                area_under, _, _ = result
-                score = area_under
+                area_under, area_from_100, _ = result
+                score = area_under if metric_type == "F+" else area_from_100
                 data_dict[col_name] = round(score, 2)
                 all_scores.append(score)
     

src/populate.py

@@ -42,14 +42,15 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
 
 
-def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list,
+                                    metric_type = "F+") -> pd.DataFrame:
     """Creates a dataframe from all the MIB experiment results"""
     # print(f"results_path is {results_path}, requests_path is {requests_path}")
     raw_data = get_raw_eval_results_mib_subgraph(results_path, requests_path)
     # print(f"raw_data is {raw_data}")
     
     # Convert each result to dict format
-    all_data_json = [v.to_dict() for v in raw_data]
+    all_data_json = [v.to_dict(metric_type=metric_type) for v in raw_data]
     # print(f"all_data_json is {pd.DataFrame.from_records(all_data_json)}")
 
     # Convert to dataframe