Merge pull request #14 from OpenGPTX/mt_bench

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 
 import core as core
-from style import CSS, T_SYMBOLS, TITLE, LANG_SYMBOLS
+from style import CSS, LANG_SYMBOLS, T_SYMBOLS, TITLE
 
 demo = gr.Blocks(css=CSS)
 with demo:
@@ -38,7 +38,7 @@ with demo:
                     )
                 with gr.Row():
                     langs_bar = gr.CheckboxGroup(
-                        choices=[(LANG_SYMBOLS.get(l,l),l) for l in core.languages_list],
+                        choices=[(LANG_SYMBOLS.get(l, l), l) for l in core.languages_list],
                         value=core.languages_list,
                         label="Select languages to average over",
                         elem_id="column-select",
@@ -52,13 +52,11 @@ with demo:
                             size="sm",
                             scale=1,
                         )
-                        select = gr.Button(
-                            value="Select all languages", size="sm", scale=1
-                        )
+                        select = gr.Button(value="Select all languages", size="sm", scale=1)
 
                         def update_bar():
                             langs_bar = gr.CheckboxGroup(
-                                choices=[(LANG_SYMBOLS.get(l,l),l) for l in core.languages_list],
+                                choices=[(LANG_SYMBOLS.get(l, l), l) for l in core.languages_list],
                                 value=core.languages_list,
                                 label="Select languages to average over",
                                 elem_id="column-select",
@@ -83,14 +81,10 @@ with demo:
                         label="Select evaluation type",
                         scale=29,
                     )
-                    clear = gr.ClearButton(
-                        shown_tasks, value="Deselect all tasks", size="sm", scale=21
-                    )
+                    clear = gr.ClearButton(shown_tasks, value="Deselect all tasks", size="sm", scale=21)
 
         with gr.Tabs(elem_classes="tab-buttons") as tabs:
-            with gr.TabItem(
-                "🏅 LLM accuracy benchmark", elem_id="llm-benchmark-tab-table-acc", id=0
-            ) as acc:
+            with gr.TabItem("🏅 LLM accuracy benchmark", elem_id="llm-benchmark-tab-table-acc", id=0) as acc:
                 leaderboard_table = gr.Dataframe()
             with gr.TabItem(
                 "🌐 LLM translation benchmark",
@@ -98,6 +92,12 @@ with demo:
                 id=1,
             ) as misc:
                 leaderboard_table_misc = gr.Dataframe()
+            with gr.TabItem(
+                "🌐 LLM MT-Bench benchmark",
+                elem_id="llm-benchmark-tab-table-mtbench",
+                id=2,
+            ) as mtbench:
+                leaderboard_table_mtbench = gr.Dataframe()
 
         demo.load(
             core.update_task_groups_and_fewshot,
@@ -119,6 +119,11 @@ with demo:
             inputs=[gr.State(value=1), fewshot],
             outputs=[shown_tasks, fewshot, selected_tab],
         )
+        mtbench.select(
+            core.update_task_groups_and_fewshot,
+            inputs=[gr.State(value=2), fewshot],
+            outputs=[shown_tasks, fewshot, selected_tab],
+        )
         for comp, fn in [
             (search_bar, "submit"),
             (langs_bar, "change"),
@@ -136,7 +141,11 @@ with demo:
                 [shown_tasks, search_bar, langs_bar, model_types, fewshot],
                 leaderboard_table_misc,
             )
-
+            getattr(comp, fn)(
+                core.update_df,
+                [shown_tasks, search_bar, langs_bar, model_types, fewshot],
+                leaderboard_table_mtbench,
+            )
 
     gr.Blocks.load(
         block=demo,
@@ -152,4 +161,11 @@ with demo:
         outputs=leaderboard_table_misc,
     )
 
+    gr.Blocks.load(
+        block=demo,
+        fn=core.update_df,
+        inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot],
+        outputs=leaderboard_table_mtbench,
+    )
+
 demo.launch()

core.py

@@ -9,7 +9,7 @@ from datasets import load_dataset
 
 import style
 
-ZERO_SHOT_ONLY = ["BELEBELE"]
+ZERO_SHOT_ONLY = ["BELEBELE", "MT-Bench"]
 FEW_SHOT_ONLY = ["GSM8K", "TruthfulQA"]
 
 
@@ -141,6 +141,11 @@ def update_task_groups_and_fewshot(current_selected_tab: int, is_fewshot_current
     elif current_selected_tab == 1:
         is_fewshot_new = False
         fewshot_available = False
+    elif current_selected_tab == 2:
+        is_fewshot_new = False
+        fewshot_available = False
+    else:
+        raise ValueError(f"Unknown tab id {current_selected_tab}")
 
     fewshot_radio_update = gr.Radio(
         value=is_fewshot_new,
@@ -151,7 +156,7 @@ def update_task_groups_and_fewshot(current_selected_tab: int, is_fewshot_current
 
 
 def get_selected_task_type(task_type_id):
-    task_types = {0: "accuracy", 1: "misc"}
+    task_types = {0: "accuracy", 1: "misc", 2: "mtbench_score"}
     selected_task_type = task_types[task_type_id]
     return selected_task_type