solved group and singular problem hierarchially

app.py

@@ -29,7 +29,7 @@ from src.display.utils import (
     AutoEvalColumnGroup,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, TOKEN, QUEUE_REPO, REPO_ID, RESULTS_REPO, EVAL_RESULTS_GROUP_PATH, RESULTS_GROUP_REPO
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.populate import get_evaluation_queue_df, get_leaderboard_df, get_evaluation_queue_df_group, get_leaderboard_group_df
 from src.submission.submit import add_new_eval
 
 
@@ -65,7 +65,7 @@ except Exception:
 
 
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
-raw_data_grouped, original_df_grouped = get_leaderboard_df(EVAL_RESULTS_GROUP_PATH, COLS_GROUP, BENCHMARK_COLS_GROUP)
+raw_data_grouped, original_df_grouped = get_leaderboard_group_df(EVAL_RESULTS_GROUP_PATH, COLS_GROUP, BENCHMARK_COLS_GROUP)
 
 leaderboard_grouped_df = original_df_grouped.copy()
 leaderboard_df = original_df.copy()
@@ -81,7 +81,7 @@ leaderboard_df = original_df.copy()
     finished_eval_queue_g_df,
     running_eval_queue_g_df,
     pending_eval_queue_g_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS_GROUP)
+) = get_evaluation_queue_df_group(EVAL_REQUESTS_PATH, EVAL_COLS_GROUP)
 
 # Searching and filtering
 def update_table(

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, Tasks
+from src.display.utils import AutoEvalColumn, Tasks, Groups
 
 @dataclass
 class EvalResult:
@@ -40,7 +40,6 @@ class EvalResult:
         # Extract results available in this file (some results are split in several files)
         results = {}
         for task in Tasks:
-            task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
@@ -72,11 +71,85 @@ class EvalResult:
         }
 
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            data_dict[task.col_name] = self.results[task.benchmark]
 
         return data_dict
 
 
+
+
+@dataclass
+class EvalResultGroup:
+    eval_name: str # org_model_date (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    results: dict
+    date: str = "" # submission date of request file
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Get model and org
+        org_and_model = config.get("model_name", None)
+        org_and_model = org_and_model.split("/", 1)
+
+        org = org_and_model[0]
+        model = org_and_model[1]
+        date = config.get("submitted_time", None)
+        result_key = f"{org}_{model}_{date}"
+        full_model = "/".join(org_and_model)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Groups:
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            date=date
+        )
+
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Groups)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.model_submission_date.name: self.date,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.average.name: average,
+        }
+
+        for task in Groups:
+            data_dict[task.col_name] = self.results[task.benchmark]
+
+        return data_dict
+
+
+
+
+
+
+
+
 def get_raw_eval_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
@@ -110,3 +183,41 @@ def get_raw_eval_results(results_path: str) -> list[EvalResult]:
             continue
 
     return results
+
+
+
+
+def get_group_eval_results(results_path: str) -> list[EvalResultGroup]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResultGroup.init_from_json_file(model_result_filepath)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results
+

src/populate.py

@@ -4,8 +4,8 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, AutoEvalColumnGroup, EvalQueueColumnGroup
+from src.leaderboard.read_evals import get_raw_eval_results, get_group_eval_results
 
 
 def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -21,6 +21,21 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
     return raw_data, df
 
 
+def get_leaderboard_group_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    raw_data = get_group_eval_results(results_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumnGroup.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return raw_data, df
+
+
+
+
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
@@ -54,3 +69,41 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
     
     return df_finished[cols], df_running[cols], df_pending[cols]
+
+
+
+
+
+def get_evaluation_queue_df_group(save_path: str, cols: list) -> list[pd.DataFrame]:
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumnGroup.model.name] = make_clickable_model(data["model"])
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumnGroup.model.name] = make_clickable_model(data["model"])
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] == "PENDING"]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"] == "FINISHED"]
+    
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    
+    return df_finished[cols], df_running[cols], df_pending[cols]