add self-reported results

app.py

@@ -52,7 +52,7 @@ with gr.Blocks() as block:
                 )
 
             # Define different captions for each table
-            default_caption = "**Table 1: MEGA-Bench full results.** The number in the parentheses is the number of tasks of each keyword. <br> The Core set contains $N_{\\text{core}} = 440$ tasks evaluated by rule-based metrics, and the Open-ended set contains $N_{\\text{open}} = 65$ tasks evaluated by a VLM judge (we use GPT-4o-0806). <br> Different from the results in our paper, we only use the Core results with CoT prompting here for clarity and compatibility with the released data. <br> $\\text{Overall} \\ = \\ \\frac{\\text{Core} \\ \\cdot \\ N_{\\text{core}} \\ + \\ \\text{Open-ended} \\ \\cdot \\ N_{\\text{open}}}{N_{\\text{core}} \\ + \\ N_{\\text{open}}}$ " 
+            default_caption = "**Table 1: MEGA-Bench full results.** The number in the parentheses is the number of tasks of each keyword. <br> The Core set contains $N_{\\text{core}} = 440$ tasks evaluated by rule-based metrics, and the Open-ended set contains $N_{\\text{open}} = 65$ tasks evaluated by a VLM judge (we use GPT-4o-0806). <br> Different from the results in our paper, we only use the Core results with CoT prompting here for clarity and compatibility with the released data. <br> $\\text{Overall} \\ = \\ \\frac{\\text{Core} \\ \\cdot \\ N_{\\text{core}} \\ + \\ \\text{Open-ended} \\ \\cdot \\ N_{\\text{open}}}{N_{\\text{core}} \\ + \\ N_{\\text{open}}}$ <br> * indicates self-reported results from the model authors."
 
             single_image_caption = "**Table 2: MEGA-Bench Single-image setting results.** The number in the parentheses is the number of tasks in each keyword. <br> This subset contains 273 single-image tasks from the Core set and 42 single-image tasks from the Open-ended set. For open-source models, we drop the image input in the 1-shot demonstration example so that the entire query contains a single image only. <br> Compared to the default table, some models with only single-image support are added."
 

constants.py

@@ -134,6 +134,10 @@ MODEL_NAME_MAP = {
     "Gemini-exp-1206": "Gemini-exp-1206",
     "Ivy_VL_3B": "Ivy-VL-3B",
     "DeepSeek_VL2_tiny": "deepseek-vl2-tiny",
+    "MiniMax-VL-01": "MiniMax-VL-01",
+    "Qwen2.5-VL-72B": "Qwen2.5-VL-72B",
+    "Qwen2.5-VL-7B": "Qwen2.5-VL-7B",
+    "Qwen2.5-VL-3B": "Qwen2.5-VL-3B",
 }
 
 DIMENSION_NAME_MAP = {
@@ -227,15 +231,19 @@ MODEL_URLS = {
     "Gemini-Flash-2.0-exp": "https://blog.google/technology/google-deepmind/google-gemini-ai-update-december-2024/#gemini-2-0-flash",
     "Ivy_VL_3B": "https://huggingface.co/AI-Safeguard/Ivy-VL-llava",
     "DeepSeek_VL2_tiny": "https://huggingface.co/deepseek-ai/deepseek-vl2-tiny",
+    "MiniMax-VL-01": "https://huggingface.co/MiniMaxAI/MiniMax-VL-01",
+    "Qwen2.5-VL-72B": "https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct",
+    "Qwen2.5-VL-7B": "https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct",
+    "Qwen2.5-VL-3B": "https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct",
 }
 
 # Define the base MODEL_GROUPS structure
 BASE_MODEL_GROUPS = {
     "All": list(MODEL_NAME_MAP.keys()),
-    "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', 'Molmo_72B', 'InternVL2_5_78B', 'Grok-2-vision-1212', "Gemini-exp-1206"],
-    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Gemini-2.0-thinking", "Gemini-Flash-2.0-exp", "Ivy_VL_3B", "DeepSeek_VL2_tiny"],
+    "Flagship Models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', 'Molmo_72B', 'InternVL2_5_78B', 'Grok-2-vision-1212', "Gemini-exp-1206", "MiniMax-VL-01", "Qwen2.5-VL-72B",],
+    "Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', 'Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Gemini-2.0-thinking", "Gemini-Flash-2.0-exp", "Ivy_VL_3B", "DeepSeek_VL2_tiny", "Qwen2.5-VL-7B", "Qwen2.5-VL-3B"],
     "Proprietary Flagship models": ['Claude_3.5_new', 'GPT_4o', 'Claude_3.5', 'Gemini_1.5_pro_002', 'Grok-2-vision-1212', "Gemini-exp-1206"],
     "Proprietary Efficiency Models": ['Gemini_1.5_flash_002', 'GPT_4o_mini', "Gemini-Flash-2.0-exp", "Gemini-2.0-thinking"],
-    "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', "Molmo_72B", "InternVL2_5_78B"],
-    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Ivy_VL_3B", "DeepSeek_VL2_tiny"]
+    "Open-source Flagship Models": ['Qwen2_VL_72B', 'InternVL2_76B', 'llava_onevision_72B', 'NVLM', "Molmo_72B", "InternVL2_5_78B", "MiniMax-VL-01", "Qwen2.5-VL-72B"],
+    "Open-source Efficiency Models": ['Qwen2_VL_7B', 'Pixtral_12B', 'Aria', 'InternVL2_8B', 'Phi-3.5-vision', 'MiniCPM_v2.6', 'llava_onevision_7B', 'Llama_3_2_11B', 'Idefics3', 'Molmo_7B_D', "Aquila_VL_2B", "POINTS_7B", "Qwen2_VL_2B", "InternVL2_2B", "InternVL2_5_2B", "InternVL2_5_8B", "Ivy_VL_3B", "DeepSeek_VL2_tiny", "Qwen2.5-VL-7B", "Qwen2.5-VL-3B"]
 }

static/eval_results/Default/self_reported.json

@@ -0,0 +1,6 @@
+{
+    "MiniMax-VL-01": 47.4,
+    "Qwen2.5-VL-72B": 51.3,
+    "Qwen2.5-VL-7B": 36.8,
+    "Qwen2.5-VL-3B": 28.9
+}

utils.py

@@ -2,19 +2,16 @@ import pandas as pd
 import json
 from typing import Dict, Any, Tuple
 import os
-from constants import (
-    MODEL_NAME_MAP,
-    DIMENSION_NAME_MAP,
-    KEYWORD_NAME_MAP,
-    MODEL_URLS,
-    BASE_MODEL_GROUPS
-)
+from constants import MODEL_NAME_MAP, DIMENSION_NAME_MAP, KEYWORD_NAME_MAP, MODEL_URLS, BASE_MODEL_GROUPS
+
 
 class MEGABenchEvalDataLoader:
     def __init__(self, base_path):
         self.base_path = base_path
         # Load both model and summary data at once
         self.KEYWORD_DATA, self.SUMMARY_DATA = self._load_data()
+        # Add loading of self-reported results
+        self.SELF_REPORTED = self._load_self_reported()
         self.SUPER_GROUPS = self._initialize_super_groups()
         self.MODEL_GROUPS = self._initialize_model_groups()
 
@@ -33,44 +30,56 @@ class MEGABenchEvalDataLoader:
                     keyword_data[model_name] = data["keyword_stats"]
                 if "model_summary" in data:
                     summary_data[model_name] = data["model_summary"]
-        
+
         return keyword_data, summary_data
 
+    def _load_self_reported(self) -> Dict[str, float]:
+        try:
+            with open(os.path.join(self.base_path, "self_reported.json"), "r") as f:
+                return json.load(f)
+        except FileNotFoundError:
+            print(
+                "Warning: No self-reported file found at",
+                os.path.join(os.path.dirname(self.base_path), "self_reported.json"),
+            )
+            return {}
+
     def _initialize_super_groups(self):
         # Get a sample model to access the structure
         sample_model = next(iter(self.KEYWORD_DATA))
-        
+
         # Create groups with task counts
         groups = {}
         self.keyword_display_map = {}  # Add this map to store display-to-original mapping
-        
+
         for dim in self.KEYWORD_DATA[sample_model]:
             dim_name = DIMENSION_NAME_MAP[dim]
             # Create a list of tuples (display_name, count, keyword) for sorting
             keyword_info = []
-            
+
             for keyword in self.KEYWORD_DATA[sample_model][dim]:
                 # Get the task count for this keyword
                 task_count = self.KEYWORD_DATA[sample_model][dim][keyword]["count"]
                 original_name = KEYWORD_NAME_MAP.get(keyword, keyword)
                 display_name = f"{original_name}({task_count})"
                 keyword_info.append((display_name, task_count, keyword))
-            
+
             # Sort by count (descending) and then by display name (for ties)
             keyword_info.sort(key=lambda x: (-x[1], x[0]))
-            
+
             # Store sorted display names and update mapping
             groups[dim_name] = [info[0] for info in keyword_info]
             for display_name, _, keyword in keyword_info:
                 self.keyword_display_map[display_name] = keyword
-        
+
         # Sort based on predefined order
         order = ["Application", "Skills", "Output Format", "Input Format", "Visual Input Number"]
         return {k: groups[k] for k in order if k in groups}
 
     def _initialize_model_groups(self) -> Dict[str, list]:
-        available_models = set(self.KEYWORD_DATA.keys())
-        
+        # Include both evaluated and self-reported models
+        available_models = set(self.KEYWORD_DATA.keys()) | set(self.SELF_REPORTED.keys())
+
         filtered_groups = {}
         for group_name, models in BASE_MODEL_GROUPS.items():
             if group_name == "All":
@@ -79,49 +88,75 @@ class MEGABenchEvalDataLoader:
                 filtered_models = [model for model in models if model in available_models]
                 if filtered_models:
                     filtered_groups[group_name] = filtered_models
-        
+
         return filtered_groups
 
     def get_df(self, selected_super_group: str, selected_model_group: str) -> pd.DataFrame:
         original_dimension = get_original_dimension(selected_super_group)
         data = []
-        
+
         for model in self.MODEL_GROUPS[selected_model_group]:
-            if model not in self.KEYWORD_DATA or model not in self.SUMMARY_DATA:
+            if (model not in self.KEYWORD_DATA or model not in self.SUMMARY_DATA) and model not in self.SELF_REPORTED:
                 continue
             
-            model_data = self.KEYWORD_DATA[model]
-            summary = self.SUMMARY_DATA[model]
-            
             # Basic model information
             row = {
                 "Models": get_display_model_name(model, as_link=True),
-                "Overall": round(summary["overall_score"] * 100, 2),
-                "Core": round(summary["core"]["macro_mean_score"] * 100, 2),
-                "Open-ended": round(summary["open"]["macro_mean_score"] * 100, 2)
             }
             
-            # Add dimension-specific scores
-            if original_dimension in model_data:
-                for display_name in self.SUPER_GROUPS[selected_super_group]:
-                    original_keyword = self.keyword_display_map[display_name]
-                    if original_keyword in model_data[original_dimension]:
-                        row[display_name] = round(model_data[original_dimension][original_keyword]["average_score"] * 100, 2)
-                    else:
-                        row[display_name] = None
-            else:
+            # Add asterisk for self-reported results
+            if model in self.SELF_REPORTED:
+                # Store numeric value for sorting but display with asterisk
+                row["Overall"] = self.SELF_REPORTED[model]
+                row["Overall_display"] = f"{self.SELF_REPORTED[model]:.2f}*"
+                row["Core"] = None
+                row["Open-ended"] = None
                 for display_name in self.SUPER_GROUPS[selected_super_group]:
                     row[display_name] = None
+            else:
+                model_data = self.KEYWORD_DATA[model]
+                summary = self.SUMMARY_DATA[model]
+                
+                # Store numeric values
+                overall_score = round(summary["overall_score"] * 100, 2)
+                row["Overall"] = overall_score
+                row["Overall_display"] = f"{overall_score:.2f}"
+                row["Core"] = round(summary["core"]["macro_mean_score"] * 100, 2)
+                row["Open-ended"] = round(summary["open"]["macro_mean_score"] * 100, 2)
                 
+                # Add dimension-specific scores
+                if original_dimension in model_data:
+                    for display_name in self.SUPER_GROUPS[selected_super_group]:
+                        original_keyword = self.keyword_display_map[display_name]
+                        if original_keyword in model_data[original_dimension]:
+                            row[display_name] = round(
+                                model_data[original_dimension][original_keyword]["average_score"] * 100, 2
+                            )
+                        else:
+                            row[display_name] = None
+                else:
+                    for display_name in self.SUPER_GROUPS[selected_super_group]:
+                        row[display_name] = None
+
             data.append(row)
-        
+
         df = pd.DataFrame(data)
+        # Sort by numeric Overall column
         df = df.sort_values(by="Overall", ascending=False)
+        
+        # Replace None with "-" for display
+        display_cols = ["Core", "Open-ended"] + self.SUPER_GROUPS[selected_super_group]
+        df[display_cols] = df[display_cols].fillna("-")
+        
+        # Replace Overall with Overall_display
+        df["Overall"] = df["Overall_display"]
+        df = df.drop("Overall_display", axis=1)
+        
         return df
 
     def get_leaderboard_data(self, selected_super_group: str, selected_model_group: str) -> Tuple[list, list]:
         df = self.get_df(selected_super_group, selected_model_group)
-        
+
         # Get total task counts from the first model's data
         sample_model = "GPT_4o"
         total_core_tasks = self.SUMMARY_DATA[sample_model]["core"]["num_eval_tasks"]
@@ -134,39 +169,42 @@ class MEGABenchEvalDataLoader:
             "Models": "Models",
             "Overall": f"Overall\n({total_tasks})",
             "Core": f"Core\n({total_core_tasks})",
-            "Open-ended": f"Open-ended\n({total_open_tasks})"
+            "Open-ended": f"Open-ended\n({total_open_tasks})",
         }
-        
+
         # Add rank column to DataFrame
         df = df.reset_index(drop=True)
-        df.insert(0, 'Rank', range(1, len(df) + 1))
-        
+        df.insert(0, "Rank", range(1, len(df) + 1))
+
         # Rename the columns in DataFrame to match headers
         df = df.rename(columns=column_headers)
-        
+
         # For dimension columns, add task counts on new line
         dimension_headers = []
         for display_name in self.SUPER_GROUPS[selected_super_group]:
-            task_count = display_name.split('(')[1].rstrip(')')
-            base_name = display_name.split('(')[0]
+            task_count = display_name.split("(")[1].rstrip(")")
+            base_name = display_name.split("(")[0]
             dimension_headers.append(f"{base_name}\n({task_count})")
-        
+
         headers = [
             column_headers["Rank"],
             column_headers["Models"],
             column_headers["Overall"],
             column_headers["Core"],
-            column_headers["Open-ended"]
+            column_headers["Open-ended"],
         ] + dimension_headers
-        
-        data = df[[
-            column_headers["Rank"],
-            column_headers["Models"],
-            column_headers["Overall"],
-            column_headers["Core"],
-            column_headers["Open-ended"]
-        ] + self.SUPER_GROUPS[selected_super_group]].values.tolist()
-        
+
+        data = df[
+            [
+                column_headers["Rank"],
+                column_headers["Models"],
+                column_headers["Overall"],
+                column_headers["Core"],
+                column_headers["Open-ended"],
+            ]
+            + self.SUPER_GROUPS[selected_super_group]
+        ].values.tolist()
+
         return headers, data
 
 
@@ -174,12 +212,13 @@ class MEGABenchEvalDataLoader:
 def get_original_dimension(mapped_dimension):
     return next(k for k, v in DIMENSION_NAME_MAP.items() if v == mapped_dimension)
 
+
 def get_original_keyword(mapped_keyword):
     return next((k for k, v in KEYWORD_NAME_MAP.items() if v == mapped_keyword), mapped_keyword)
 
+
 def get_display_model_name(model_name: str, as_link: bool = True) -> str:
     display_name = MODEL_NAME_MAP.get(model_name, model_name)
     if as_link and model_name in MODEL_URLS:
         return f'<a href="{MODEL_URLS[model_name]}" target="_blank" style="text-decoration: none; color: #2196F3;">{display_name}</a>'
     return display_name
-