Display llm-jp-eval version and backend library

app.py

@@ -26,9 +26,11 @@ from src.display.utils import (
     TYPES,
     AddSpecialTokens,
     AutoEvalColumn,
+    Backend,
     ModelType,
     NumFewShots,
     Precision,
+    Version,
     fields,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO
@@ -75,6 +77,8 @@ def filter_models(
     precision_query: list,
     add_special_tokens_query: list,
     num_few_shots_query: list,
+    version_query: list,
+    backend_query: list,
 ) -> pd.DataFrame:
     print(f"Initial df shape: {df.shape}")
     print(f"Initial df content:\n{df}")
@@ -110,6 +114,14 @@ def filter_models(
     ]
     print(f"After num_few_shots filter: {filtered_df.shape}")
 
+    # Version フィルタリング
+    filtered_df = filtered_df[filtered_df["llm-jp-eval version"].isin(version_query)]
+    print(f"After version filter: {filtered_df.shape}")
+
+    # Backend フィルタリング
+    filtered_df = filtered_df[filtered_df["Backend Library"].isin(backend_query)]
+    print(f"After backend filter: {filtered_df.shape}")
+
     print("Filtered dataframe head:")
     print(filtered_df.head())
     return filtered_df
@@ -177,6 +189,8 @@ def update_table(
     size_query: list,
     add_special_tokens_query: list,
     num_few_shots_query: list,
+    version_query: list,
+    backend_query: list,
     query: str,
 ):
     print(
@@ -191,6 +205,8 @@ def update_table(
         precision_query,
         add_special_tokens_query,
         num_few_shots_query,
+        version_query,
+        backend_query,
     )
     print(f"filtered_df shape after filter_models: {filtered_df.shape}")
 
@@ -236,6 +252,8 @@ leaderboard_df = filter_models(
     [i.value.name for i in Precision],
     [i.value.name for i in AddSpecialTokens],
     [i.value.name for i in NumFewShots],
+    [i.value.name for i in Version],
+    [i.value.name for i in Backend],
 )
 
 leaderboard_df_filtered = filter_models(
@@ -245,6 +263,8 @@ leaderboard_df_filtered = filter_models(
     [i.value.name for i in Precision],
     [i.value.name for i in AddSpecialTokens],
     [i.value.name for i in NumFewShots],
+    [i.value.name for i in Version],
+    [i.value.name for i in Backend],
 )
 
 # DataFrameの初期化部分のみを修正
@@ -309,6 +329,18 @@ with gr.Blocks() as demo_leaderboard:
                 value=[i.value.name for i in NumFewShots],
                 elem_id="filter-columns-num-few-shots",
             )
+            filter_columns_version = gr.CheckboxGroup(
+                label="Version",
+                choices=[i.value.name for i in Version],
+                value=[i.value.name for i in Version],
+                elem_id="filter-columns-version",
+            )
+            filter_columns_backend = gr.CheckboxGroup(
+                label="Backend",
+                choices=[i.value.name for i in Backend],
+                value=[i.value.name for i in Backend],
+                elem_id="filter-columns-backend",
+            )
 
     # DataFrameコンポーネントの初期化
     leaderboard_table = gr.Dataframe(
@@ -340,6 +372,8 @@ with gr.Blocks() as demo_leaderboard:
             filter_columns_size.change,
             filter_columns_add_special_tokens.change,
             filter_columns_num_few_shots.change,
+            filter_columns_version.change,
+            filter_columns_backend.change,
             search_bar.submit,
         ],
         fn=update_table,
@@ -351,6 +385,8 @@ with gr.Blocks() as demo_leaderboard:
             filter_columns_size,
             filter_columns_add_special_tokens,
             filter_columns_num_few_shots,
+            filter_columns_version,
+            filter_columns_backend,
             search_bar,
         ],
         outputs=leaderboard_table,

src/display/utils.py

@@ -44,6 +44,10 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 auto_eval_column_dict.append(["num_few_shots", ColumnContent, ColumnContent("Few-shot", "str", False)])
 auto_eval_column_dict.append(["add_special_tokens", ColumnContent, ColumnContent("Add Special Tokens", "bool", False)])
+auto_eval_column_dict.append(
+    ["llm_jp_eval_version", ColumnContent, ColumnContent("llm-jp-eval version", "str", False)]
+)
+auto_eval_column_dict.append(["backend", ColumnContent, ColumnContent("Backend Library", "str", False)])
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
@@ -130,6 +134,28 @@ class NumFewShots(Enum):
         return NumFewShots.Unknown
 
 
+class Version(Enum):
+    v1_4_1 = ModelDetails("v1.4.1")
+    Unknown = ModelDetails("?")
+
+    def from_str(version):
+        if version == "1.4.1":
+            return Version.v1_4_1
+        else:
+            return Version.Unknown
+
+
+class Backend(Enum):
+    vllm = ModelDetails("vllm")
+    Unknown = ModelDetails("?")
+
+    def from_str(backend):
+        if backend == "vllm":
+            return Backend.vllm
+        else:
+            return Backend.Unknown
+
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn)]

src/leaderboard/read_evals.py

@@ -7,7 +7,7 @@ from decimal import Decimal
 import dateutil
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, WeightType
+from src.display.utils import AutoEvalColumn, Backend, ModelType, Tasks, Version, WeightType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -34,6 +34,8 @@ class EvalResult:
     still_on_hub: bool = False
     num_few_shots: str = "0"
     add_special_tokens: str = ""
+    llm_jp_eval_version: str = ""
+    backend: str = ""
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -62,6 +64,9 @@ class EvalResult:
             config.get("pipeline_kwargs", {"add_special_tokens": "Unknown"}).get("add_special_tokens")
         )
 
+        version = Version.from_str(metainfo.get("version", "?")).value.name
+        backend = Backend.from_str(model_config.get("_target_", "?").split(".")[0]).value.name
+
         # Get model and org
         # org_and_model = config.get("model_name", config.get("offline_inference").get("model_name", None))
         org_and_model = config.get("model_name", config.get("offline_inference", {}).get("model_name", "Unknown"))
@@ -116,6 +121,8 @@ class EvalResult:
             architecture=architecture,
             num_few_shots=num_few_shots,
             add_special_tokens=add_special_tokens,
+            llm_jp_eval_version=version,
+            backend=backend,
         )
 
     def update_with_request_file(self, requests_path):
@@ -153,6 +160,8 @@ class EvalResult:
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
             AutoEvalColumn.num_few_shots.name: self.num_few_shots,
             AutoEvalColumn.add_special_tokens.name: self.add_special_tokens,
+            AutoEvalColumn.llm_jp_eval_version.name: self.llm_jp_eval_version,
+            AutoEvalColumn.backend.name: self.backend,
         }
 
         # for task in Tasks: