New leaderboard design (#19)

* MT-BENCH: Model type is now fixed at "chat" for MT-BENCH. Pretrained models are not shown nor can be selected.
* MT-BENCH: Language selection in MT-BENCH tab is limited to EN, DE, ES, FR, IT
* MT-BENCH: Don't select all 22 Languages when "Select all languages" button is pressed in in Mt-Bench tab.
* New Leaderboard Design: New design skeleton
* New Leaderboard Design: Removed unnecessary updates
* New Leaderboard Design: Introduced Zero-Shot tab instead of radio buttons

---------

Co-authored-by: KlaudiaTH <[email protected]>

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 
 import core as core
-from style import CSS, LANG_SYMBOLS, T_SYMBOLS, TITLE, MT_BENCH_LANG_SYMBOLS
+from style import CSS, LANG_SYMBOLS, MT_BENCH_LANG_SYMBOLS, T_SYMBOLS, TITLE
 
 demo = gr.Blocks(css=CSS)
 with demo:
@@ -14,8 +14,12 @@ with demo:
 
     selected_tab = gr.State(value=0)
 
-    with gr.Column():
-        with gr.Row():
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem(
+            "🏅 LLM accuracy benchmark",
+            elem_id="llm-benchmark-tab-table-acc",
+            id=0,
+        ) as acc:
             with gr.Column():
                 with gr.Row():
                     search_bar = gr.Textbox(
@@ -24,7 +28,6 @@ with demo:
                         show_label=True,
                         elem_id="search-bar",
                     )
-
                     model_types = gr.CheckboxGroup(
                         label="Select model type",
                         choices=[
@@ -36,6 +39,7 @@ with demo:
                         ],
                         value=list(T_SYMBOLS.values()),
                     )
+
                 with gr.Row():
                     langs_bar = gr.CheckboxGroup(
                         choices=[(LANG_SYMBOLS.get(l, l), l) for l in core.languages_list],
@@ -52,125 +56,318 @@ with demo:
                             size="sm",
                             scale=1,
                         )
-                        select = gr.Button(value="Select all languages", size="sm", scale=1)
-
-                        def update_bar(selected_tab):
-                            if selected_tab == 2:
-                                choices = [(MT_BENCH_LANG_SYMBOLS.get(l, l), l) for l in core.mt_bench_language_list]
-                                value = core.mt_bench_language_list
-                            else:
-                                choices = [(LANG_SYMBOLS.get(l, l), l) for l in core.languages_list]
-                                value = core.languages_list
-                            langs_bar = gr.CheckboxGroup(
-                                choices=choices,
-                                value=value,
-                                label="Select languages to average over",
-                                elem_id="column-select",
-                                interactive=True,
-                            )
-                            return langs_bar
-
-                        select.click(update_bar, inputs=[selected_tab], outputs=langs_bar)
+                        select = gr.Button(
+                            value="Select all languages",
+                            size="sm",
+                            scale=1,
+                        )
+                        select.click(
+                            lambda: gr.CheckboxGroup(value=core.languages_list),
+                            inputs=[],
+                            outputs=langs_bar,
+                        )
+
+            with gr.Row():
+                shown_tasks = gr.CheckboxGroup(
+                    choices=core.get_available_task_groups(core.get_selected_task_type(0), True),
+                    value=core.get_available_task_groups(core.get_selected_task_type(0), True),
+                    label="Select tasks to show",
+                    elem_id="column-select",
+                    interactive=True,
+                    scale=50,
+                )
+                clear = gr.ClearButton(
+                    shown_tasks,
+                    value="Deselect all tasks",
+                    size="sm",
+                    scale=1,
+                )
+                select = gr.Button(
+                    value="Select all tasks",
+                    size="sm",
+                    scale=1,
+                )
+                select.click(
+                    lambda: gr.CheckboxGroup(value=core.get_available_task_groups(core.get_selected_task_type(0), True)),
+                    inputs=[],
+                    outputs=shown_tasks,
+                )
+            leaderboard_table = gr.Dataframe()
+
+        with gr.TabItem(
+            "🏅 LLM accuracy benchmark (Zero-Shot)",
+            elem_id="llm-benchmark-tab-table-acc-zeroshot",
+            id=3,
+        ) as acc_zero_shot:
+            with gr.Column():
+                with gr.Row():
+                    search_bar_zero_shot = gr.Textbox(
+                        label="Search models",
+                        placeholder=" 🔍 Separate multiple queries with ';' and press ENTER...",
+                        show_label=True,
+                        elem_id="search-bar",
+                    )
+                    model_types_zero_shot = gr.CheckboxGroup(
+                        label="Select model type",
+                        choices=[
+                            (
+                                f"Pretrained {T_SYMBOLS['pretrained']}",
+                                T_SYMBOLS["pretrained"],
+                            ),
+                            (f"Chat {T_SYMBOLS['chat']}", T_SYMBOLS["chat"]),
+                        ],
+                        value=list(T_SYMBOLS.values()),
+                    )
+
+                with gr.Row():
+                    langs_bar_zero_shot = gr.CheckboxGroup(
+                        choices=[(LANG_SYMBOLS.get(l, l), l) for l in core.languages_list],
+                        value=core.languages_list,
+                        label="Select languages to average over",
+                        elem_id="column-select",
+                        interactive=True,
+                        scale=6,
+                    )
+                    with gr.Column(scale=1):
+                        clear_zero_shot = gr.ClearButton(
+                            langs_bar_zero_shot,
+                            value="Deselect all languages",
+                            size="sm",
+                            scale=1,
+                        )
+                        select_zero_shot = gr.Button(
+                            value="Select all languages",
+                            size="sm",
+                            scale=1,
+                        )
+                        select_zero_shot.click(
+                            lambda: gr.CheckboxGroup(value=core.languages_list),
+                            inputs=[],
+                            outputs=langs_bar_zero_shot,
+                        )
+
+            with gr.Row():
+                shown_tasks_zero_shot = gr.CheckboxGroup(
+                    choices=core.get_available_task_groups(core.get_selected_task_type(3), False),
+                    value=core.get_available_task_groups(core.get_selected_task_type(3), False),
+                    label="Select tasks to show",
+                    elem_id="column-select",
+                    interactive=True,
+                    scale=50,
+                )
+                clear_zero_shot = gr.ClearButton(
+                    shown_tasks_zero_shot,
+                    value="Deselect all tasks",
+                    size="sm",
+                    scale=1,
+                )
+                select_zero_shot = gr.Button(
+                    value="Select all tasks",
+                    size="sm",
+                    scale=1,
+                )
+                select_zero_shot.click(
+                    lambda: gr.CheckboxGroup(value=core.get_available_task_groups(core.get_selected_task_type(3), False)),
+                    inputs=[],
+                    outputs=shown_tasks_zero_shot,
+                )
+            leaderboard_table_zero_shot = gr.Dataframe()
+
+        with gr.TabItem(
+            "🌐 LLM translation benchmark",
+            elem_id="llm-benchmark-tab-table-misc",
+            id=1,
+        ) as misc:
+            with gr.Column():
+                with gr.Row():
+                    search_bar_misc = gr.Textbox(
+                        label="Search models",
+                        placeholder=" 🔍 Separate multiple queries with ';' and press ENTER...",
+                        show_label=True,
+                        elem_id="search-bar",
+                    )
+
+                    model_types_misc = gr.CheckboxGroup(
+                        label="Select model type",
+                        choices=[
+                            (
+                                f"Pretrained {T_SYMBOLS['pretrained']}",
+                                T_SYMBOLS["pretrained"],
+                            ),
+                            (f"Chat {T_SYMBOLS['chat']}", T_SYMBOLS["chat"]),
+                        ],
+                        value=list(T_SYMBOLS.values()),
+                    )
 
                 with gr.Row():
-                    shown_tasks = gr.CheckboxGroup(
-                        choices=[],
-                        value=[],
-                        label="Select tasks to show",
+                    langs_bar_misc = gr.CheckboxGroup(
+                        choices=[(LANG_SYMBOLS.get(l, l), l) for l in core.languages_list],
+                        value=core.languages_list,
+                        label="Select languages to average over",
                         elem_id="column-select",
                         interactive=True,
-                        scale=50,
+                        scale=6,
+                    )
+                    with gr.Column(scale=1):
+                        clear_misc = gr.ClearButton(
+                            langs_bar_misc,
+                            value="Deselect all languages",
+                            size="sm",
+                            scale=1,
+                        )
+                        select_misc = gr.Button(
+                            value="Select all languages",
+                            size="sm",
+                            scale=1,
+                        )
+                        select_misc.click(
+                            lambda: gr.CheckboxGroup(value=core.languages_list),
+                            inputs=[],
+                            outputs=langs_bar_misc,
+                        )
+
+            with gr.Row():
+                shown_tasks_misc = gr.CheckboxGroup(
+                    choices=core.get_available_task_groups(core.get_selected_task_type(1), False),
+                    value=core.get_available_task_groups(core.get_selected_task_type(1), False),
+                    label="Select tasks to show",
+                    elem_id="column-select",
+                    interactive=True,
+                    scale=50,
+                )
+                clear_tasks_misc = gr.ClearButton(
+                    shown_tasks_misc,
+                    value="Deselect all tasks",
+                    size="sm",
+                    scale=1,
+                )
+                select_all_tasks_misc = gr.Button(
+                    value="Select all tasks",
+                    size="sm",
+                    scale=1,
+                )
+                select_all_tasks_misc.click(
+                    lambda: gr.CheckboxGroup(value=core.get_available_task_groups(core.get_selected_task_type(1), False)),
+                    inputs=[],
+                    outputs=shown_tasks_misc,
+                )
+
+            leaderboard_table_misc = gr.Dataframe()
+
+        with gr.TabItem(
+            "🌐 LLM MT-Bench benchmark",
+            elem_id="llm-benchmark-tab-table-mtbench",
+            id=2,
+        ) as mtbench:
+            with gr.Column():
+                with gr.Row():
+                    search_bar_mtbench = gr.Textbox(
+                        label="Search models",
+                        placeholder=" 🔍 Separate multiple queries with ';' and press ENTER...",
+                        show_label=True,
+                        elem_id="search-bar",
+                    )
+
+            with gr.Row():
+                langs_bar_mtbench = gr.CheckboxGroup(
+                    choices=[(LANG_SYMBOLS.get(l, l), l) for l in core.mt_bench_language_list],
+                    value=core.mt_bench_language_list,
+                    label="Select languages to average over",
+                    elem_id="column-select",
+                    interactive=True,
+                    scale=6,
+                )
+                with gr.Column(scale=1):
+                    clear_mtbench = gr.ClearButton(
+                        langs_bar_mtbench,
+                        value="Deselect all languages",
+                        size="sm",
+                        scale=1,
                     )
-                    fewshot = gr.Radio(
-                        choices=[("0-Shot", False), ("Few-shot", True)],
-                        value=True,
-                        label="Select evaluation type",
-                        scale=29,
+                    select_mtbench = gr.Button(
+                        value="Select all languages",
+                        size="sm",
+                        scale=1,
                     )
-                    clear = gr.ClearButton(shown_tasks, value="Deselect all tasks", size="sm", scale=21)
-
-        with gr.Tabs(elem_classes="tab-buttons") as tabs:
-            with gr.TabItem("🏅 LLM accuracy benchmark", elem_id="llm-benchmark-tab-table-acc", id=0) as acc:
-                leaderboard_table = gr.Dataframe()
-            with gr.TabItem(
-                "🌐 LLM translation benchmark",
-                elem_id="llm-benchmark-tab-table-misc",
-                id=1,
-            ) as misc:
-                leaderboard_table_misc = gr.Dataframe()
-            with gr.TabItem(
-                "🌐 LLM MT-Bench benchmark",
-                elem_id="llm-benchmark-tab-table-mtbench",
-                id=2,
-            ) as mtbench:
-                leaderboard_table_mtbench = gr.Dataframe()
-
-        demo.load(
-            core.update_task_groups_and_fewshot,
-            [gr.State(value=0), model_types, langs_bar,fewshot],
-            [shown_tasks, fewshot, selected_tab, model_types, langs_bar],
-        )
-        fewshot.change(
-            core.update_task_groups_and_fewshot,
-            [selected_tab, model_types, langs_bar, fewshot],
-            [shown_tasks, fewshot, selected_tab, model_types, langs_bar],
-        )
-        acc.select(
-            core.update_task_groups_and_fewshot,
-            inputs=[gr.State(value=0), model_types, langs_bar, fewshot],
-            outputs=[shown_tasks, fewshot, selected_tab, model_types, langs_bar],
-        )
-        misc.select(
-            core.update_task_groups_and_fewshot,
-            inputs=[gr.State(value=1), model_types, langs_bar, fewshot],
-            outputs=[shown_tasks, fewshot, selected_tab, model_types, langs_bar],
-        )
-        mtbench.select(
-            core.update_task_groups_and_fewshot,
-            inputs=[gr.State(value=2), model_types, langs_bar, fewshot],
-            outputs=[shown_tasks, fewshot, selected_tab, model_types, langs_bar],
-        )
+                    select_mtbench.click(
+                        lambda: gr.CheckboxGroup(value=core.mt_bench_language_list),
+                        inputs=[],
+                        outputs=langs_bar_mtbench,
+                    )
+
+            leaderboard_table_mtbench = gr.Dataframe(scale=5)
+
         for comp, fn in [
             (search_bar, "submit"),
             (langs_bar, "change"),
             (shown_tasks, "change"),
-            (fewshot, "change"),
             (model_types, "change"),
         ]:
             getattr(comp, fn)(
                 core.update_df,
-                [shown_tasks, search_bar, langs_bar, model_types, fewshot],
+                [shown_tasks, search_bar, langs_bar, model_types, gr.State(value=True)],
                 leaderboard_table,
             )
+
+        for comp, fn in [
+            (search_bar_zero_shot, "submit"),
+            (model_types_zero_shot, "change"),
+            (langs_bar_zero_shot, "change"),
+            (shown_tasks_zero_shot, "change"),
+        ]:
             getattr(comp, fn)(
                 core.update_df,
-                [shown_tasks, search_bar, langs_bar, model_types, fewshot],
+                [shown_tasks_zero_shot, search_bar_zero_shot, langs_bar_zero_shot, model_types_zero_shot, gr.State(value=False)],
+                leaderboard_table_zero_shot,
+            )
+
+        for comp, fn in [
+            (search_bar_misc, "submit"),
+            (langs_bar_misc, "change"),
+            (shown_tasks_misc, "change"),
+            (model_types_misc, "change"),
+        ]:
+            getattr(comp, fn)(
+                core.update_df,
+                [shown_tasks_misc, search_bar_misc, langs_bar_misc, model_types_misc, gr.State(value=False)],
                 leaderboard_table_misc,
             )
+
+        for comp, fn in [
+            (search_bar_mtbench, "submit"),
+            (langs_bar_mtbench, "change"),
+        ]:
             getattr(comp, fn)(
                 core.update_df,
-                [shown_tasks, search_bar, langs_bar, model_types, fewshot],
+                [gr.State(value=core.get_available_task_groups(core.get_selected_task_type(2), False)), search_bar_mtbench, langs_bar_mtbench, gr.State(value=[T_SYMBOLS["chat"]]), gr.State(value=False)],  # TODO
                 leaderboard_table_mtbench,
             )
 
     gr.Blocks.load(
         block=demo,
         fn=core.update_df,
-        inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot],
+        inputs=[shown_tasks, search_bar, langs_bar, model_types, gr.State(value=True)],
         outputs=leaderboard_table,
     )
 
     gr.Blocks.load(
         block=demo,
         fn=core.update_df,
-        inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot],
+        inputs=[shown_tasks_zero_shot, search_bar_zero_shot, langs_bar_zero_shot, model_types_zero_shot, gr.State(value=False)],
+        outputs=leaderboard_table_zero_shot,
+    )
+
+    gr.Blocks.load(
+        block=demo,
+        fn=core.update_df,
+        inputs=[shown_tasks_misc, search_bar_misc, langs_bar_misc, model_types_misc, gr.State(value=False)],
         outputs=leaderboard_table_misc,
     )
 
     gr.Blocks.load(
         block=demo,
         fn=core.update_df,
-        inputs=[shown_tasks, search_bar, langs_bar, model_types, fewshot],
+        inputs=[gr.State(value=core.get_available_task_groups(core.get_selected_task_type(2), False)), search_bar_mtbench, langs_bar_mtbench, gr.State(value=[T_SYMBOLS["chat"]]), gr.State(value=False)],
         outputs=leaderboard_table_mtbench,
     )
 

core.py

@@ -1,13 +1,11 @@
 import itertools
 import os
 
-import gradio as gr
 import numpy as np
 import pandas as pd
 from datasets import load_dataset
 
 import style
-from style import T_SYMBOLS, MT_BENCH_LANG_SYMBOLS, LANG_SYMBOLS
 
 ZERO_SHOT_ONLY = ["BELEBELE", "MT-Bench"]
 FEW_SHOT_ONLY = ["GSM8K", "TruthfulQA"]
@@ -29,7 +27,7 @@ def init():
     task_groups_shots_df = hidden_df[hidden_df["Few_Shot"] == True][["Task_Group", "Number_Shots"]].drop_duplicates()
     task_groups_shots_dict = task_groups_shots_df.set_index("Task_Group")["Number_Shots"].to_dict()
     languages_list = hidden_df["Language"].drop_duplicates().str.upper().tolist()
-    mt_bench_language_list = hidden_df[hidden_df['Task_Group'] == "MTBENCH"]["Language"].drop_duplicates().str.upper().tolist()
+    mt_bench_language_list = hidden_df[hidden_df["Task_Group"] == "MTBENCH"]["Language"].drop_duplicates().str.upper().tolist()
     model_type_df = hidden_df[["Model_Name", "Model_Type"]].drop_duplicates()
     model_type_dict = model_type_df.set_index("Model_Name")["Model_Type"].to_dict()
 
@@ -115,8 +113,7 @@ def update_df(
 
     # aggregate results over languages per task
     df = aggregate_langs(df, tasks, langs)
-
-    df = df.sort_values(by='Average', ascending=False)
+    df = df.sort_values(by="Average", ascending=False)
 
     # filter models by search bar and model type
     df = search_model(df, model_query)
@@ -128,67 +125,8 @@ def update_df(
         return sort_cols(df, fewshot)
 
 
-def update_task_groups_and_fewshot(current_selected_tab: int, model_types, langs_bar, is_fewshot_current: bool = False, ):
-    selected_task_type = get_selected_task_type(current_selected_tab)
-    available_tasks = get_available_task_groups(selected_task_type, is_fewshot_current)
-    new_selected_tasks = available_tasks.copy()
-
-    tasks_checkbox_group_update = gr.CheckboxGroup(
-        choices=available_tasks,
-        value=new_selected_tasks,
-    )
-
-    if current_selected_tab == 0:
-        is_fewshot_new = is_fewshot_current
-        fewshot_available = True
-    elif current_selected_tab == 1:
-        is_fewshot_new = False
-        fewshot_available = False
-    elif current_selected_tab == 2:
-        is_fewshot_new = False
-        fewshot_available = False
-    else:
-        raise ValueError(f"Unknown tab id {current_selected_tab}")
-
-    fewshot_radio_update = gr.Radio(
-        value=is_fewshot_new,
-        interactive=fewshot_available,
-    )
-
-    if current_selected_tab == 2:
-        model_types = gr.CheckboxGroup(
-            value=[T_SYMBOLS['chat']],
-            interactive=False
-        )
-        langs_bar = gr.CheckboxGroup(
-            choices=[(MT_BENCH_LANG_SYMBOLS.get(l, l), l) for l in mt_bench_language_list],
-            value=mt_bench_language_list,
-            interactive=True,
-        )
-    else:
-        model_types = gr.CheckboxGroup(
-            label="Select model type",
-            choices=[
-                (
-                    f"Pretrained {T_SYMBOLS['pretrained']}",
-                    T_SYMBOLS["pretrained"],
-                ),
-                (f"Chat {T_SYMBOLS['chat']}", T_SYMBOLS["chat"]),
-            ],
-            value=list(T_SYMBOLS.values()),
-            interactive=True
-        )
-        langs_bar = gr.CheckboxGroup(
-            choices=[(LANG_SYMBOLS.get(l, l), l) for l in languages_list],
-            value=languages_list,
-            interactive=True,
-        )
-
-    return [tasks_checkbox_group_update, fewshot_radio_update, current_selected_tab, model_types, langs_bar]
-
-
 def get_selected_task_type(task_type_id):
-    task_types = {0: "accuracy", 1: "misc", 2: "mtbench_score"}
+    task_types = {0: "accuracy", 1: "misc", 2: "mtbench_score", 3: "accuracy"}
     selected_task_type = task_types[task_type_id]
     return selected_task_type
 

style.py

@@ -11,10 +11,101 @@ CSS = """
 }
 """
 
-T_SYMBOLS = {
-    "pretrained": "🟢",
-    "chat": "💬"
+OPEN_LLM_LEADERBOARD_CSS = """
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
 }
+/* Full width space */
+.gradio-container {
+    max-width: 95% !important;
+}
+/* Text style and margins */
+.markdown-text {
+    font-size: 16px !important;
+}
+#models-to-add-text {
+    font-size: 18px !important;
+}
+#citation-button span {
+    font-size: 16px !important;
+}
+#citation-button textarea {
+    font-size: 16px !important;
+}
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+#search-bar {
+    padding: 0px;
+}
+.tab-buttons button {
+    font-size: 20px;
+}
+/* Filters style */
+#filter_type {
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span {
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap {
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner {
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input {
+    width: 1px;
+}
+#filter-columns-type {
+    border: 0;
+    padding: 0.5;
+}
+#filter-columns-size {
+    border: 0;
+    padding: 0.5;
+}
+#box-filter > .form {
+    border: 0;
+}
+/* Header styles */
+#header-title {
+    text-align: left;
+    display: inline-block;
+}
+#header-row {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+#header-row .gradio-html {
+    flex-grow: 1;
+}
+#oauth-button {
+    height: auto;
+    min-width: max-content;
+    white-space: nowrap;
+    padding: 10px 20px;
+    border-radius: 4px;
+}
+"""
+
+T_SYMBOLS = {"pretrained": "🟢", "chat": "💬"}
 
 LANG_SYMBOLS = {
     "BG": "🇧🇬 BG",
@@ -37,13 +128,7 @@ LANG_SYMBOLS = {
     "RO": "🇷🇴 RO",
     "SK": "🇸🇰 SK",
     "SL": "🇸🇮 SL",
-    "SV": "🇸🇪 SV"
+    "SV": "🇸🇪 SV",
 }
 
-MT_BENCH_LANG_SYMBOLS = {
-    "ES": "🇪🇸 ES",
-    "EN": "🇬🇧 EN",
-    "DE": "🇩🇪 DE",
-    "FR": "🇫🇷 FR",
-    "IT": "🇮🇹 IT"
-}
+MT_BENCH_LANG_SYMBOLS = {"ES": "🇪🇸 ES", "EN": "🇬🇧 EN", "DE": "🇩🇪 DE", "FR": "🇫🇷 FR", "IT": "🇮🇹 IT"}