add vllm version's filter

app.py

@@ -34,6 +34,7 @@ from src.display.utils import (
     NumFewShots,
     Precision,
     Version,
+    VllmVersion,
     fields,
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO
@@ -108,6 +109,7 @@ def filter_models(
     add_special_tokens_query: list,
     num_few_shots_query: list,
     version_query: list,
+    vllm_query: list,
     # backend_query: list,
 ) -> pd.DataFrame:
     print(f"Initial df shape: {df.shape}")
@@ -146,6 +148,10 @@ def filter_models(
     filtered_df = filtered_df[filtered_df["llm-jp-eval version"].isin(version_query)]
     print(f"After version filter: {filtered_df.shape}")
 
+    # Vllm Version フィルタリング
+    filtered_df = filtered_df[filtered_df["vllm version"].isin(vllm_query)]
+    print(f"After vllm version filter: {filtered_df.shape}")
+
     # Backend フィルタリング
     # filtered_df = filtered_df[filtered_df["Backend Library"].isin(backend_query)]
     # print(f"After backend filter: {filtered_df.shape}")
@@ -199,6 +205,7 @@ def update_table(
     add_special_tokens_query: list,
     num_few_shots_query: list,
     version_query: list,
+    vllm_query: list,
     # backend_query: list,
     query: str,
     *columns,
@@ -216,6 +223,7 @@ def update_table(
         add_special_tokens_query,
         num_few_shots_query,
         version_query,
+        vllm_query,
         #    backend_query,
     )
     print(f"filtered_df shape after filter_models: {filtered_df.shape}")
@@ -248,6 +256,7 @@ leaderboard_df = filter_models(
     [i.value.name for i in AddSpecialTokens],
     [i.value.name for i in NumFewShots],
     [i.value.name for i in Version],
+    [i.value.name for i in VllmVersion],
     #    [i.value.name for i in Backend],
 )
 # DataFrameの初期化部分のみを修正
@@ -492,6 +501,13 @@ with gr.Blocks() as demo_leaderboard:
                 value=[i.value.name for i in Version],
                 elem_id="filter-columns-version",
             )
+            filter_columns_vllm = gr.CheckboxGroup(
+                label="vllm version",
+                choices=[i.value.name for i in VllmVersion],
+                value=[i.value.name for i in VllmVersion],
+                elem_id="filter-columns-vllm",
+            )
+
             # filter_columns_backend = gr.CheckboxGroup(
             #    label="Backend Library",
             #    choices=[i.value.name for i in Backend],
@@ -539,6 +555,7 @@ with gr.Blocks() as demo_leaderboard:
             filter_columns_add_special_tokens.change,
             filter_columns_num_few_shots.change,
             filter_columns_version.change,
+            filter_columns_vllm.change,
             # filter_columns_backend.change,
             search_bar.submit,
         ]
@@ -551,6 +568,7 @@ with gr.Blocks() as demo_leaderboard:
             filter_columns_add_special_tokens,
             filter_columns_num_few_shots,
             filter_columns_version,
+            filter_columns_vllm,
             # filter_columns_backend,
             search_bar,
         ]

src/display/utils.py

@@ -60,6 +60,7 @@ auto_eval_column_dict.append(["add_special_tokens", ColumnContent, ColumnContent
 auto_eval_column_dict.append(
     ["llm_jp_eval_version", ColumnContent, ColumnContent("llm-jp-eval version", "str", False)]
 )
+auto_eval_column_dict.append(["vllm_version", ColumnContent, ColumnContent("vllm version", "str", False)])
 auto_eval_column_dict.append(["backend", ColumnContent, ColumnContent("Backend Library", "str", False, dummy=True)])
 auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
 auto_eval_column_dict.append(["row_id", ColumnContent, ColumnContent("ID", "number", False, dummy=True)])

src/leaderboard/read_evals.py

@@ -34,6 +34,7 @@ class EvalResult:
     num_few_shots: str = "0"
     add_special_tokens: str = ""
     llm_jp_eval_version: str = ""
+    vllm_version: str = ""
     backend: str = ""
 
     @classmethod
@@ -67,6 +68,10 @@ class EvalResult:
         )
 
         version = Version.from_str(metainfo.get("version", "?")).value.name
+
+        # Get vllm version from metainfo
+        vllm_version = metainfo.get("vllm-version", "")
+
         backend = Backend.from_str(model_config.get("_target_", "?").split(".")[0]).value.name
         revision = model_config.get("revision", "")
 
@@ -114,6 +119,7 @@ class EvalResult:
             num_few_shots=num_few_shots,
             add_special_tokens=add_special_tokens,
             llm_jp_eval_version=version,
+            vllm_version=vllm_version,
             backend=backend,
         )
 
@@ -153,6 +159,7 @@ class EvalResult:
             AutoEvalColumn.num_few_shots.name: self.num_few_shots,
             AutoEvalColumn.add_special_tokens.name: self.add_special_tokens,
             AutoEvalColumn.llm_jp_eval_version.name: self.llm_jp_eval_version,
+            AutoEvalColumn.vllm_version.name: self.vllm_version,
             AutoEvalColumn.backend.name: self.backend,
         }