Add hidden option

src/leaderboard/read_evals.py

@@ -38,6 +38,7 @@ class EvalResult:
     json_filename: str = None
     eval_time: float = 0.0
     original_benchmark_average: float = None
+    hidden: bool = False # Do not show on the leaderboard
 
     @classmethod
     def init_from_json_file(self, json_filepath, is_original=False):
@@ -125,6 +126,7 @@ class EvalResult:
             self.date = request.get("submitted_time", "")
             self.architecture = request.get("architectures", "Unknown")
             self.status = request.get("status", "FAILED")
+            self.hidden = request.get("hidden", False)
         except Exception as e:
             self.status = "FAILED"
             print(f"Could not find request file for {self.org}/{self.model}")
@@ -163,7 +165,7 @@ class EvalResult:
             AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
             AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),
             AutoEvalColumn.flagged.name: self.flagged,
-            AutoEvalColumn.eval_time.name: self.eval_time,
+            AutoEvalColumn.eval_time.name: self.eval_time
         }
 
         for task in Tasks:
@@ -236,7 +238,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, dynamic_path: st
     results = []
     for v in eval_results.values():
         try:
-            if v.status == "FINISHED":
+            if v.status == "FINISHED" and not v.hidden:
                 v.to_dict() # we test if the dict version is complete
                 results.append(v)
         except KeyError:  # not all eval values present