add some uncommited code of f3f40fb

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from utils import DefaultDataLoader, SingleImageDataLoader
+from utils import MEGABenchEvalDataLoader
 import os
 from constants import *
 
@@ -20,8 +20,8 @@ with open(si_css_file, "r") as f:
     si_css = f.read()
 
 # Initialize data loaders
-default_loader = DefaultDataLoader()
-si_loader = SingleImageDataLoader()
+default_loader = MEGABenchEvalDataLoader("./static/eval_results/Default")
+si_loader = MEGABenchEvalDataLoader("./static/eval_results/SI")
 
 with gr.Blocks() as block:
     # Add a style element that we'll update

utils.py

@@ -10,29 +10,48 @@ from constants import (
     BASE_MODEL_GROUPS
 )
 
-class BaseDataLoader:
-    def __init__(self):
-        self.MODEL_DATA = self._load_model_data()
-        self.SUMMARY_DATA = self._load_summary_data()
+class MEGABenchEvalDataLoader:
+    def __init__(self, base_path):
+        self.base_path = base_path
+        # Load both model and summary data at once
+        self.KEYWORD_DATA, self.SUMMARY_DATA = self._load_data()
         self.SUPER_GROUPS = self._initialize_super_groups()
         self.MODEL_GROUPS = self._initialize_model_groups()
 
+    def _get_base_path(self) -> str:
+        raise NotImplementedError("Subclasses must implement _get_base_path")
+
+    def _load_data(self) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+        summary_data = {}
+        keyword_data = {}
+        model_folders = [f for f in os.listdir(self.base_path) if os.path.isdir(os.path.join(self.base_path, f))]
+        for model_name in model_folders:
+            model_path = f"{self.base_path}/{model_name}/summary_and_keyword_stats.json"
+            with open(model_path, "r") as f:
+                data = json.load(f)
+                if "keyword_stats" in data:
+                    keyword_data[model_name] = data["keyword_stats"]
+                if "model_summary" in data:
+                    summary_data[model_name] = data["model_summary"]
+        
+        return keyword_data, summary_data
+
     def _initialize_super_groups(self):
         # Get a sample model to access the structure
-        sample_model = next(iter(self.MODEL_DATA))
+        sample_model = next(iter(self.KEYWORD_DATA))
         
         # Create groups with task counts
         groups = {}
         self.keyword_display_map = {}  # Add this map to store display-to-original mapping
         
-        for dim in self.MODEL_DATA[sample_model]:
+        for dim in self.KEYWORD_DATA[sample_model]:
             dim_name = DIMENSION_NAME_MAP[dim]
             # Create a list of tuples (display_name, count, keyword) for sorting
             keyword_info = []
             
-            for keyword in self.MODEL_DATA[sample_model][dim]:
+            for keyword in self.KEYWORD_DATA[sample_model][dim]:
                 # Get the task count for this keyword
-                task_count = self.MODEL_DATA[sample_model][dim][keyword]["count"]
+                task_count = self.KEYWORD_DATA[sample_model][dim][keyword]["count"]
                 original_name = KEYWORD_NAME_MAP.get(keyword, keyword)
                 display_name = f"{original_name}({task_count})"
                 keyword_info.append((display_name, task_count, keyword))
@@ -50,7 +69,7 @@ class BaseDataLoader:
         return {k: groups[k] for k in order if k in groups}
 
     def _initialize_model_groups(self) -> Dict[str, list]:
-        available_models = set(self.MODEL_DATA.keys())
+        available_models = set(self.KEYWORD_DATA.keys())
         
         filtered_groups = {}
         for group_name, models in BASE_MODEL_GROUPS.items():
@@ -63,21 +82,15 @@ class BaseDataLoader:
         
         return filtered_groups
 
-    def _load_model_data(self) -> Dict[str, Any]:
-        raise NotImplementedError("Subclasses must implement _load_model_data")
-
-    def _load_summary_data(self) -> Dict[str, Any]:
-        raise NotImplementedError("Subclasses must implement _load_summary_data")
-
     def get_df(self, selected_super_group: str, selected_model_group: str) -> pd.DataFrame:
         original_dimension = get_original_dimension(selected_super_group)
         data = []
         
         for model in self.MODEL_GROUPS[selected_model_group]:
-            if model not in self.MODEL_DATA or model not in self.SUMMARY_DATA:
+            if model not in self.KEYWORD_DATA or model not in self.SUMMARY_DATA:
                 continue
             
-            model_data = self.MODEL_DATA[model]
+            model_data = self.KEYWORD_DATA[model]
             summary = self.SUMMARY_DATA[model]
             
             # Basic model information
@@ -110,11 +123,11 @@ class BaseDataLoader:
         df = self.get_df(selected_super_group, selected_model_group)
         
         # Get total task counts from the first model's data
-        sample_model = next(iter(self.SUMMARY_DATA))
+        sample_model = "GPT_4o"
         total_core_tasks = self.SUMMARY_DATA[sample_model]["core"]["num_eval_tasks"]
         total_open_tasks = self.SUMMARY_DATA[sample_model]["open"]["num_eval_tasks"]
         total_tasks = total_core_tasks + total_open_tasks
-        
+
         # Define headers with task counts
         column_headers = {
             "Models": "Models",
@@ -143,84 +156,6 @@ class BaseDataLoader:
         return headers, data
 
 
-class DefaultDataLoader(BaseDataLoader):
-    def __init__(self):
-        super().__init__()
-
-    def _load_model_data(self) -> Dict[str, Any]:
-        model_data = {}
-        base_path = "./static/eval_results/Default"
-        
-        try:
-            model_folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))]
-            for model_name in model_folders:
-                model_path = f"{base_path}/{model_name}/summary_results.json"
-                with open(model_path, "r") as f:
-                    data = json.load(f)
-                    if "keyword_stats" in data:
-                        model_data[model_name] = data["keyword_stats"]
-        except FileNotFoundError:
-            pass
-        
-        return model_data
-
-    def _load_summary_data(self) -> Dict[str, Any]:
-        summary_data = {}
-        base_path = "./static/eval_results/Default"
-        
-        try:
-            model_folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))]
-            for model_name in model_folders:
-                model_path = f"{base_path}/{model_name}/summary_results.json"
-                with open(model_path, "r") as f:
-                    data = json.load(f)
-                    if "model_summary" in data:
-                        summary_data[model_name] = data["model_summary"]
-        except FileNotFoundError:
-            pass
-        
-        return summary_data
-
-
-class SingleImageDataLoader(BaseDataLoader):
-    def __init__(self):
-        super().__init__()
-
-    def _load_model_data(self) -> Dict[str, Any]:
-        model_data = {}
-        base_path = "./static/eval_results/SI"
-        
-        try:
-            model_folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))]
-            for model_name in model_folders:
-                model_path = f"{base_path}/{model_name}/summary_results.json"
-                with open(model_path, "r") as f:
-                    data = json.load(f)
-                    if "keyword_stats" in data:
-                        model_data[model_name] = data["keyword_stats"]
-        except FileNotFoundError:
-            pass
-        
-        return model_data
-
-    def _load_summary_data(self) -> Dict[str, Any]:
-        summary_data = {}
-        base_path = "./static/eval_results/SI"
-        
-        try:
-            model_folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))]
-            for model_name in model_folders:
-                model_path = f"{base_path}/{model_name}/summary_results.json"
-                with open(model_path, "r") as f:
-                    data = json.load(f)
-                    if "model_summary" in data:
-                        summary_data[model_name] = data["model_summary"]
-        except FileNotFoundError:
-            pass
-        
-        return summary_data
-
-
 # Keep your helper functions
 def get_original_dimension(mapped_dimension):
     return next(k for k, v in DIMENSION_NAME_MAP.items() if v == mapped_dimension)