debug

app.py

@@ -74,7 +74,7 @@ except Exception:
 
 LEADERBOARD_DF_MIB_SUBGRAPH = get_leaderboard_df_mib_subgraph(EVAL_RESULTS_MIB_SUBGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_SUBGRAPH, BENCHMARK_COLS_MIB_SUBGRAPH)
 
-# LEADERBOARD_DF_MIB_CAUSALGRAPH = get_leaderboard_df_mib_causalgraph(EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_CAUSALGRAPH, BENCHMARK_COLS_MIB_CAUSALGRAPH)
+LEADERBOARD_DF_MIB_CAUSALGRAPH = get_leaderboard_df_mib_causalgraph(EVAL_RESULTS_MIB_CAUSALGRAPH_PATH, EVAL_REQUESTS_PATH, COLS_MIB_CAUSALGRAPH, BENCHMARK_COLS_MIB_CAUSALGRAPH)
 
 # LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 # LEADERBOARD_DF_MULTIMODAL = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS_MULTIMODAL, BENCHMARK_COLS_MULTIMODAL)
@@ -203,8 +203,8 @@ with demo:
         with gr.TabItem("Subgraph", elem_id="subgraph", id=0):
             leaderboard = init_leaderboard_mib_subgraph(LEADERBOARD_DF_MIB_SUBGRAPH, "Subgraph")
         
-        # with gr.TabItem("Causal Graph", elem_id="causalgraph", id=1):
-        #     leaderboard = init_leaderboard_mib_causalgraph(LEADERBOARD_DF_MIB_CAUSALGRAPH, "Causal Graph")
+        with gr.TabItem("Causal Graph", elem_id="causalgraph", id=1):
+            leaderboard = init_leaderboard_mib_causalgraph(LEADERBOARD_DF_MIB_CAUSALGRAPH, "Causal Graph")
         
     # with gr.Row():
     #     with gr.Accordion("📙 Citation", open=False):

src/about.py

@@ -8,13 +8,6 @@ class Task:
     col_name: str
 
 
-@dataclass
-class TaskMIB:
-    benchmark: str      # task name in json (ioi/arithmetic)
-    models: list[str]   # list of models to show as sub-columns
-    col_name: str       # display name in leaderboard
-    metrics: list[str]  # metrics to store (edge_counts, faithfulness)
-
 
 
 # Select your tasks here
@@ -27,10 +20,6 @@ class Tasks(Enum):
     task3 = Task("ewok", "acc", "EWoK")
 
 
-class TasksMib_Subgraph(Enum):
-    task0 = TaskMIB("ioi", ["meta_llama", "qwen", "gpt2"], "ioi", ["edge_counts", "faithfulness"])
-    task1 = TaskMIB("mcqa", ["meta_llama", "qwen", "gpt2"], "mcqa", ["edge_counts", "faithfulness"])
-
     
 class TasksMultimodal(Enum):
     task0 = Task("blimp", "acc", "BLiMP")
@@ -41,6 +30,44 @@ class TasksMultimodal(Enum):
     task5 = Task("winoground", "acc", "Winoground")
     task6 = Task("devbench", "acc", "DevBench")
 
+
+
+@dataclass
+class TaskMIB_Subgraph:
+    benchmark: str      # task name in json (ioi/arithmetic)
+    models: list[str]   # list of models to show as sub-columns
+    col_name: str       # display name in leaderboard
+    metrics: list[str]  # metrics to store (edge_counts, faithfulness)
+
+class TasksMib_Subgraph(Enum):
+    task0 = TaskMIB_Subgraph("ioi", ["meta_llama", "qwen", "gpt2"], "ioi", ["edge_counts", "faithfulness"])
+    task1 = TaskMIB_Subgraph("mcqa", ["meta_llama", "qwen", "gpt2"], "mcqa", ["edge_counts", "faithfulness"])
+
+
+@dataclass 
+class TaskMIB_Causalgraph:
+    benchmark: str      # MCQA
+    models: list[str]   # LlamaForCausalLM
+    layers: list[str]   # 0-31
+    col_name: str       # display name in leaderboard
+    interventions: list[str]  # output_token, output_location
+    counterfactuals: list[str]  # symbol_counterfactual, randomLetter_counterfactual, etc.
+    metrics: list[str]  # score <- Added this field
+
+class TasksMib_Causalgraph(Enum):
+    task0 = TaskMIB_Causalgraph(
+        "MCQA", 
+        ["LlamaForCausalLM"],
+        [str(i) for i in range(32)],  # 0-31 layers
+        "mcqa",
+        ["output_token", "output_location"],
+        ["symbol_counterfactual", "randomLetter_counterfactual", 
+         "answerPosition_counterfactual", "answerPosition_symbol_counterfactual"],
+        ["score"]  # Added this
+    )
+
+
+
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks, TasksMultimodal, TasksMib_Subgraph
+from src.about import Tasks, TasksMultimodal, TasksMib_Subgraph, TasksMib_Causalgraph
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -21,10 +21,41 @@ class ColumnContent:
     never_hidden: bool = False
 
 ## Leaderboard columns
-auto_eval_column_dict_mib_subgraph = []
 auto_eval_column_dict = []
 auto_eval_column_dict_multimodal = []
 
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["hf_repo", ColumnContent, ColumnContent("HF Repo", "str", False)])
+auto_eval_column_dict.append(["track", ColumnContent, ColumnContent("Track", "markdown", False)])
+#Scores
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["text_average", ColumnContent, ColumnContent("Text Average", "number", True)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+auto_eval_column_dict_multimodal.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict_multimodal.append(["hf_repo", ColumnContent, ColumnContent("HF Repo", "str", False)])
+auto_eval_column_dict_multimodal.append(["track", ColumnContent, ColumnContent("Track", "markdown", False)])
+for task in TasksMultimodal:
+    auto_eval_column_dict_multimodal.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    if task.value.col_name in ("ewok", "EWoK"):   # make sure this appears in the right order
+        auto_eval_column_dict_multimodal.append(["text_average", ColumnContent, ColumnContent("Text Average", "number", True)])
+auto_eval_column_dict_multimodal.append(["vision_average", ColumnContent, ColumnContent("Vision Average", "number", True)])
+auto_eval_column_dict_multimodal.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict_multimodal.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+
+
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+AutoEvalColumnMultimodal = make_dataclass("AutoEvalColumnMultimodal", auto_eval_column_dict_multimodal, frozen=True)
+
+
+
+
+
+
 
 
 
@@ -67,6 +98,45 @@ COLS_MIB_CAUSALGRAPH = []
 BENCHMARK_COLS_MIB_CAUSALGRAPH = []
 
 
+# Initialize the MIB causal graph columns
+auto_eval_column_dict_mib_causalgraph = []
+
+# Method name column
+auto_eval_column_dict_mib_causalgraph.append(["method", ColumnContent, ColumnContent("Method", "markdown", True, never_hidden=True)])
+
+# For each layer-intervention-counterfactual combination
+for task in TasksMib_Causalgraph:
+    for model in task.value.models:
+        for layer in task.value.layers:
+            for intervention in task.value.interventions:
+                for counterfactual in task.value.counterfactuals:
+                    # Create column name like "layer0_output_token_symbol_counterfactual"
+                    col_name = f"layer{layer}_{intervention}_{counterfactual}"
+                    field_name = col_name.lower()
+                    auto_eval_column_dict_mib_causalgraph.append([
+                        field_name, 
+                        ColumnContent, 
+                        ColumnContent(col_name, "number", True)
+                    ])
+
+# Create the dataclass for MIB causal graph columns
+AutoEvalColumn_mib_causalgraph = make_dataclass("AutoEvalColumn_mib_causalgraph", auto_eval_column_dict_mib_causalgraph, frozen=True)
+
+# Column selection for display
+COLS_MIB_CAUSALGRAPH = [c.name for c in fields(AutoEvalColumn_mib_causalgraph) if not c.hidden]
+BENCHMARK_COLS_MIB_CAUSALGRAPH = [f"layer{layer}_{intervention}_{counterfactual}" 
+                                 for task in TasksMib_Causalgraph
+                                 for model in task.value.models
+                                 for layer in task.value.layers
+                                 for intervention in task.value.interventions
+                                 for counterfactual in task.value.counterfactuals]
+
+
+
+
+
+
+
 
 
 
@@ -76,32 +146,10 @@ BENCHMARK_COLS_MIB_CAUSALGRAPH = []
 
 
 
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["hf_repo", ColumnContent, ColumnContent("HF Repo", "str", False)])
-auto_eval_column_dict.append(["track", ColumnContent, ColumnContent("Track", "markdown", False)])
-#Scores
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["text_average", ColumnContent, ColumnContent("Text Average", "number", True)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
-auto_eval_column_dict_multimodal.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict_multimodal.append(["hf_repo", ColumnContent, ColumnContent("HF Repo", "str", False)])
-auto_eval_column_dict_multimodal.append(["track", ColumnContent, ColumnContent("Track", "markdown", False)])
-for task in TasksMultimodal:
-    auto_eval_column_dict_multimodal.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-    if task.value.col_name in ("ewok", "EWoK"):   # make sure this appears in the right order
-        auto_eval_column_dict_multimodal.append(["text_average", ColumnContent, ColumnContent("Text Average", "number", True)])
-auto_eval_column_dict_multimodal.append(["vision_average", ColumnContent, ColumnContent("Vision Average", "number", True)])
-auto_eval_column_dict_multimodal.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict_multimodal.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 
 
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-AutoEvalColumnMultimodal = make_dataclass("AutoEvalColumnMultimodal", auto_eval_column_dict_multimodal, frozen=True)
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)

src/leaderboard/read_evals.py

@@ -182,7 +182,104 @@ def get_raw_eval_results_mib_subgraph(results_path: str, requests_path: str) ->
 
 
 
+@dataclass
+class EvalResult_MIB_CAUSALGRAPH:
+    """Represents one full evaluation for a method in MIB causalgraph."""
+    eval_name: str        # method name as identifier
+    method_name: str      # name of the interpretation method (e.g., "baseline_patching")
+    results: Dict         # nested dict of results {model_id: {task_scores: [{layer, scores}]}}
+
+    def init_from_json_file(self, json_filepath):
+        """Inits results from the method result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        method_name = data.get("method_name")
+        results = {}
+        
+        # Get results for each model
+        for model_result in data.get("results", []):
+            model_id = model_result.get("model_id", "")
+            task_scores = model_result.get("task_scores", {})
+            
+            # Process MCQA task scores
+            mcqa_scores = {}
+            for layer_data in task_scores.get("MCQA", []):
+                layer = layer_data.get("layer")
+                layer_scores = layer_data.get("layer_scores", [])
+                
+                # Store scores for each intervention and counterfactual
+                for intervention_data in layer_scores:
+                    intervention = intervention_data["intervention"][0]  # e.g., "output_token"
+                    counterfactual_scores = intervention_data["counterfactual_scores"]
+                    
+                    for cf_score in counterfactual_scores:
+                        counterfactual = cf_score["counterfactual"][0]  # e.g., "symbol_counterfactual"
+                        score = cf_score["score"]
+                        
+                        # Create key for this combination
+                        key = f"layer{layer}_{intervention}_{counterfactual}"
+                        mcqa_scores[key] = score
+
+            results[model_id] = mcqa_scores
+
+        return EvalResult_MIB_CAUSALGRAPH(
+            eval_name=method_name,
+            method_name=method_name,
+            results=results
+        )
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict for dataframe display"""
+        data_dict = {
+            "eval_name": self.eval_name,
+            "Method": self.method_name,
+        }
+
+        # For each model, add all layer/intervention/counterfactual combinations
+        for model_id, scores in self.results.items():
+            for score_key, score_value in scores.items():
+                data_dict[score_key] = score_value
+                
+        return data_dict
+
+def get_raw_eval_results_mib_causalgraph(results_path: str, requests_path: str) -> List[EvalResult_MIB_CAUSALGRAPH]:
+    """From the path of the results folder root, extract all needed info for MIB causalgraph results"""
+    model_result_filepaths = []
+    
+    print(f"results_path is {results_path}")
+    
+    for root, dirnames, files in os.walk(results_path):
+        print(f"root is {root}, dirnames is {dirnames}, files is {files}")
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+        
+        # Sort the files by date - keeping original sorting logic
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
 
+    print(f"model_result_filepaths is {model_result_filepaths}")
+
+    eval_results = []
+    for model_result_filepath in model_result_filepaths:
+        try:
+            eval_result = EvalResult_MIB_CAUSALGRAPH("", "", {})  # Create empty instance
+            result = eval_result.init_from_json_file(model_result_filepath)
+            print(f"eval_result.init_from_json_file(model_result_filepath) is {result}")
+            # Verify the result can be converted to dict format
+            result.to_dict()
+            eval_results.append(result)
+        except Exception as e:
+            print(f"Error processing {model_result_filepath}: {e}")
+            continue
+
+    return eval_results
 
 
 

src/populate.py

@@ -66,11 +66,23 @@ def get_leaderboard_df_mib_subgraph(results_path: str, requests_path: str, cols:
     return df
 
 def get_leaderboard_df_mib_causalgraph(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the MIB experiment results"""
+    """Creates a dataframe from all the MIB causal graph experiment results"""
     print(f"results_path is {results_path}, requests_path is {requests_path}")
     raw_data = get_raw_eval_results_mib_causalgraph(results_path, requests_path)
-    # Implement the rest of the code
-    return raw_data
+    print(f"raw_data is {raw_data}")
+    
+    # Convert each result to dict format
+    all_data_json = [v.to_dict() for v in raw_data]
+    print(f"all_data_json is {pd.DataFrame.from_records(all_data_json)}")
+
+    # Convert to dataframe
+    df = pd.DataFrame.from_records(all_data_json)
+    
+    # Round numeric columns to 2 decimal places
+    numeric_cols = df.select_dtypes(include=['float64', 'int64']).columns
+    df[numeric_cols] = df[numeric_cols].round(2)
+    
+    return df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: