Update space

app.py

@@ -171,7 +171,8 @@ model_types = {
     'Small Dense Retrievers': ['DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)'],
     'LLM-based Dense Retrievers': ['ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b'],
     'Multivector Retrievers': ['multi-ada-002', 'ColBERTv2'],
-    'LLM Rerankers': ['Claude3 Reranker', 'GPT4 Reranker']
+    'LLM Rerankers': ['Claude3 Reranker', 'GPT4 Reranker'],
+    'Others': []  # Will be populated dynamically with submitted models
 }
 
 # Submission form validation functions
@@ -381,21 +382,25 @@ def scan_submissions_directory():
         return None
 
 def initialize_leaderboard():
-    """
-    Initialize the leaderboard with baseline results and submitted results.
-    """
     global df_synthesized_full, df_synthesized_10, df_human_generated
     
     try:
-        # First, initialize with baseline results
+        # Initialize with baseline results
         df_synthesized_full = pd.DataFrame(data_synthesized_full)
         df_synthesized_10 = pd.DataFrame(data_synthesized_10)
         df_human_generated = pd.DataFrame(data_human_generated)
         
         print("Initialized with baseline results")
         
-        # Then scan and add submitted results
-        scan_submissions_directory()
+        # Scan submissions directory and update 'Others' category
+        submissions = scan_submissions_directory()
+        if submissions:
+            for split_submissions in submissions.values():
+                for submission in split_submissions:
+                    method_name = submission.get('Method Name')
+                    method_exists = any(method_name in models for models in model_types.values())
+                    if not method_exists:
+                        model_types['Others'].append(method_name)
         
         print("Leaderboard initialization complete")
         
@@ -577,26 +582,26 @@ def format_evaluation_results(results):
     
 def process_submission(
     method_name, team_name, dataset, split, contact_email,
-    code_repo, csv_file, model_description, hardware, paper_link
+    code_repo, csv_file, model_description, hardware, paper_link, model_type
 ):
     """Process and validate submission"""
     temp_files = []
     try:
         # Input validation
-        if not all([method_name, team_name, dataset, split, contact_email, code_repo, csv_file]):
+        if not all([method_name, team_name, dataset, split, contact_email, code_repo, csv_file, model_type]):
             return "Error: Please fill in all required fields"
         
-        # Length validation
-        if len(method_name) > 25:
-            return "Error: Method name must be 25 characters or less"
-        if len(team_name) > 25:
-            return "Error: Team name must be 25 characters or less"
-        if not validate_email(contact_email):
-            return "Error: Invalid email format"
-        if not validate_github_url(code_repo):
-            return "Error: Invalid GitHub repository URL"
-        
-        # Create metadata at the beginning to ensure it's available for error handling
+        if model_type not in model_types:
+            return "Error: Invalid model type selected"
+
+        # Add model to 'Others' category if it's a new model
+        method_exists = any(method_name in models for models in model_types.values())
+        if not method_exists and model_type != 'Others':
+            return "Error: New models must be submitted under 'Others' category"
+        elif not method_exists and model_type == 'Others':
+            model_types['Others'].append(method_name)
+
+        # Create metadata
         meta_data = {
             "Method Name": method_name,
             "Team Name": team_name,
@@ -606,7 +611,8 @@ def process_submission(
             "Code Repository": code_repo,
             "Model Description": model_description,
             "Hardware": hardware,
-            "(Optional) Paper link": paper_link
+            "(Optional) Paper link": paper_link,
+            "Model Type": model_type
         }
         
         # Generate folder name and timestamp
@@ -755,7 +761,9 @@ def process_submission(
 def filter_by_model_type(df, selected_types):
     if not selected_types:
         return df.head(0)
-    selected_models = [model for type in selected_types for model in model_types[type]]
+    selected_models = []
+    for type in selected_types:
+        selected_models.extend(model_types[type])
     return df[df['Method'].isin(selected_models)]
 
 def format_dataframe(df, dataset):
@@ -872,6 +880,11 @@ with gr.Blocks(css=css) as demo:
             )
         
         with gr.Column():
+            model_type = gr.Dropdown(
+                choices=list(model_types.keys()),
+                label="Model Type*",
+                value="Others"
+            )
             code_repo = gr.Textbox(
                 label="Code Repository*",
                 placeholder="https://github.com/snap-stanford/stark-leaderboard"
@@ -911,7 +924,7 @@ with gr.Blocks(css=css) as demo:
         fn=process_submission,
         inputs=[
             method_name, team_name, dataset, split, contact_email,
-            code_repo, csv_file, model_description, hardware, paper_link
+            code_repo, csv_file, model_description, hardware, paper_link, model_type
         ],
         outputs=result
     ).success(  # Add a success handler to update tables after successful submission