Update space

app.py

@@ -262,39 +262,81 @@ def scan_submissions_directory():
                 repo_id=REPO_ID,
                 repo_type="space"
             )
+            # Filter for files in submissions directory
             repo_files = [f for f in all_files if f.startswith('submissions/')]
         except Exception as e:
             print(f"Error listing repository contents: {str(e)}")
             return submissions_by_split
-
-        # Process submissions and update model types
+            
+        # Group files by team folders
+        folder_files = {}
         for filepath in repo_files:
-            if filepath.endswith('metadata.json'):
-                try:
-                    submission_data = read_json_from_hub(api, REPO_ID, filepath)
-                    if submission_data:
-                        method_name = submission_data.get('Method Name')
-                        model_type = submission_data.get('Model Type')
-                        
-                        # If model type is specified in metadata, use it
-                        if method_name and model_type:
-                            # Check if method exists in any other category
-                            existing_type = get_model_type_for_method(method_name)
-                            
-                            # If method doesn't exist in any category, add it to the specified category
-                            if existing_type == 'Others' and model_type in model_types:
-                                if method_name not in model_types[model_type]:
-                                    model_types[model_type].append(method_name)
-                            
-                            # Add submission to appropriate split
-                            split = submission_data.get('Split')
-                            if split in submissions_by_split:
-                                submissions_by_split[split].append(submission_data)
+            parts = filepath.split('/')
+            if len(parts) < 3:  # Need at least submissions/team_folder/file
+                continue
                 
-                except Exception as e:
-                    print(f"Error processing metadata file {filepath}: {str(e)}")
+            folder_name = parts[1]  # team_folder name
+            if folder_name not in folder_files:
+                folder_files[folder_name] = []
+            folder_files[folder_name].append(filepath)
+        
+        # Process each team folder
+        for folder_name, files in folder_files.items():
+            try:
+                # Find latest.json in this folder
+                latest_file = next((f for f in files if f.endswith('latest.json')), None)
+                if not latest_file:
+                    print(f"No latest.json found in {folder_name}")
                     continue
-
+                
+                # Read latest.json
+                latest_info = read_json_from_hub(api, REPO_ID, latest_file)
+                if not latest_info:
+                    print(f"Failed to read latest.json for {folder_name}")
+                    continue
+                
+                timestamp = latest_info.get('latest_submission')
+                if not timestamp:
+                    print(f"No timestamp found in latest.json for {folder_name}")
+                    continue
+                
+                # Find metadata file for latest submission
+                metadata_file = next(
+                    (f for f in files if f.endswith(f'metadata_{timestamp}.json')), 
+                    None
+                )
+                if not metadata_file:
+                    print(f"No matching metadata file found for {folder_name} timestamp {timestamp}")
+                    continue
+                
+                # Read metadata file
+                submission_data = read_json_from_hub(api, REPO_ID, metadata_file)
+                if not submission_data:
+                    print(f"Failed to read metadata for {folder_name}")
+                    continue
+                
+                if latest_info.get('status') != 'approved':
+                    print(f"Skipping unapproved submission in {folder_name}")
+                    continue
+                
+                # Add to submissions by split
+                split = submission_data.get('Split')
+                if split in submissions_by_split:
+                    submissions_by_split[split].append(submission_data)
+                    
+                    # Update model types if necessary
+                    method_name = submission_data.get('Method Name')
+                    model_type = submission_data.get('Model Type', 'Others')
+                    
+                    # Add to model type if it's a new method
+                    method_exists = any(method_name in methods for methods in model_types.values())
+                    if not method_exists and model_type in model_types:
+                        model_types[model_type].append(method_name)
+                
+            except Exception as e:
+                print(f"Error processing folder {folder_name}: {str(e)}")
+                continue
+                
         return submissions_by_split
         
     except Exception as e:
@@ -302,25 +344,49 @@ def scan_submissions_directory():
         return None
 
 def initialize_leaderboard():
+    """
+    Initialize the leaderboard with baseline results and submitted results.
+    """
     global df_synthesized_full, df_synthesized_10, df_human_generated
     
     try:
-        # Initialize with baseline results
+        # First, initialize with baseline results
         df_synthesized_full = pd.DataFrame(data_synthesized_full)
         df_synthesized_10 = pd.DataFrame(data_synthesized_10)
         df_human_generated = pd.DataFrame(data_human_generated)
         
         print("Initialized with baseline results")
         
-        # Scan submissions directory and update 'Others' category
+        # Then scan and add submitted results
         submissions = scan_submissions_directory()
         if submissions:
-            for split_submissions in submissions.values():
+            for split, split_submissions in submissions.items():
                 for submission in split_submissions:
-                    method_name = submission.get('Method Name')
-                    method_exists = any(method_name in models for models in model_types.values())
-                    if not method_exists:
-                        model_types['Others'].append(method_name)
+                    if submission.get('results'):  # Make sure we have results
+                        # Update appropriate DataFrame based on split
+                        if split == 'test':
+                            df_to_update = df_synthesized_full
+                        elif split == 'test-0.1':
+                            df_to_update = df_synthesized_10
+                        else:  # human_generated_eval
+                            df_to_update = df_human_generated
+                            
+                        # Prepare new row data
+                        new_row = {
+                            'Method': submission['Method Name'],
+                            f'STARK-{submission["Dataset"].upper()}_Hit@1': submission['results']['hit@1'],
+                            f'STARK-{submission["Dataset"].upper()}_Hit@5': submission['results']['hit@5'],
+                            f'STARK-{submission["Dataset"].upper()}_R@20': submission['results']['recall@20'],
+                            f'STARK-{submission["Dataset"].upper()}_MRR': submission['results']['mrr']
+                        }
+                        
+                        # Update existing row or add new one
+                        method_mask = df_to_update['Method'] == submission['Method Name']
+                        if method_mask.any():
+                            for col in new_row:
+                                df_to_update.loc[method_mask, col] = new_row[col]
+                        else:
+                            df_to_update.loc[len(df_to_update)] = new_row
         
         print("Leaderboard initialization complete")
         
@@ -705,11 +771,18 @@ def process_submission(
 
 
 def filter_by_model_type(df, selected_types):
+    """
+    Filter DataFrame by selected model types, including submitted models.
+    """
     if not selected_types:
         return df.head(0)
+        
+    # Get all models from selected types
     selected_models = []
-    for type in selected_types:
-        selected_models.extend(model_types[type])
+    for type_name in selected_types:
+        selected_models.extend(model_types[type_name])
+    
+    # Filter DataFrame to include only selected models
     return df[df['Method'].isin(selected_models)]
 
 def format_dataframe(df, dataset):