multilingual results

app.py

@@ -31,14 +31,14 @@ from src.display.utils import (
     MEDICAL_SUMMARIZATION_BENCHMARK_COLS,
     ACI_BENCHMARK_COLS,
     SOAP_BENCHMARK_COLS,
-    CLOSED_ENDED_ARABIC_BENCHMARK_COLS,
+    #CLOSED_ENDED_ARABIC_BENCHMARK_COLS,
     DATASET_COLS,
     OPEN_ENDED_COLS,
     MED_SAFETY_COLS,
     MEDICAL_SUMMARIZATION_COLS,
     ACI_COLS,
     SOAP_COLS,
-    CLOSED_ENDED_ARABIC_COLS,
+    #CLOSED_ENDED_ARABIC_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     NUMERIC_INTERVALS,
@@ -50,7 +50,23 @@ from src.display.utils import (
     Precision,
     WeightType,
     fields,
-    render_generation_templates
+    render_generation_templates,
+    OpenEndedArabic_COLS,
+    OpenEndedArabic_BENCHMARK_COLS,
+    OpenEndedFrench_COLS,
+    OpenEndedFrench_BENCHMARK_COLS,
+    OpenEndedPortuguese_COLS,
+    OpenEndedPortuguese_BENCHMARK_COLS,
+    OpenEndedRomanian_COLS,
+    OpenEndedRomanian_BENCHMARK_COLS,
+    OpenEndedGreek_COLS,
+    OpenEndedGreek_BENCHMARK_COLS,
+    OpenEndedSpanish_COLS,
+    OpenEndedSpanish_BENCHMARK_COLS,
+    ClosedEndedMultilingual_COLS,
+    ClosedEndedMultilingual_BENCHMARK_COLS,
+    
+    
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, PRIVATE_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
@@ -96,9 +112,28 @@ aci_leaderboard_df = aci_original_df.copy()
 _, soap_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, SOAP_COLS, SOAP_BENCHMARK_COLS, "score", "soap")
 soap_leaderboard_df = soap_original_df.copy()
 
-if PRIVATE_REPO:
-    _, closed_ended_arabic_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, CLOSED_ENDED_ARABIC_COLS, CLOSED_ENDED_ARABIC_BENCHMARK_COLS, "score", "closed_ended_arabic")
-    closed_ended_arabic_leaderboard_df = closed_ended_arabic_original_df.copy()
+
+_, open_ended_arabic_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedArabic_COLS, OpenEndedArabic_BENCHMARK_COLS, "score", "open_ended_arabic")
+_, open_ended_french_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedFrench_COLS, OpenEndedFrench_BENCHMARK_COLS, "score", "open_ended_french")
+_, open_ended_portuguese_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedPortuguese_COLS, OpenEndedPortuguese_BENCHMARK_COLS, "score", "open_ended_portuguese")
+_, open_ended_romanian_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedRomanian_COLS, OpenEndedRomanian_BENCHMARK_COLS, "score", "open_ended_romanian")
+_, open_ended_greek_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedGreek_COLS, OpenEndedGreek_BENCHMARK_COLS, "score", "open_ended_greek")
+_, open_ended_spanish_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedSpanish_COLS, OpenEndedSpanish_BENCHMARK_COLS, "score", "open_ended_spanish")
+_, closed_ended_multilingual_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ClosedEndedMultilingual_COLS, ClosedEndedMultilingual_BENCHMARK_COLS, "score", "closed_ended_multilingual")
+
+
+open_ended_arabic_leaderboard_df = open_ended_arabic_df.copy()
+open_ended_french_leaderboard_df = open_ended_french_df.copy()
+open_ended_portuguese_leaderboard_df = open_ended_portuguese_df.copy()
+open_ended_romanian_leaderboard_df = open_ended_romanian_df.copy()
+open_ended_greek_leaderboard_df = open_ended_greek_df.copy()
+open_ended_spanish_leaderboard_df = open_ended_spanish_df.copy()
+closed_ended_multilingual_leaderboard_df = closed_ended_multilingual_df.copy()
+
+
+# if PRIVATE_REPO:
+#     _, closed_ended_arabic_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, CLOSED_ENDED_ARABIC_COLS, CLOSED_ENDED_ARABIC_BENCHMARK_COLS, "score", "closed_ended_arabic")
+#     closed_ended_arabic_leaderboard_df = closed_ended_arabic_original_df.copy()
 
 # breakpoint()
 # # Token based results
@@ -136,9 +171,28 @@ def update_df(shown_columns, subset="datasets"):
     elif subset == "soap":
         leaderboard_table_df = soap_leaderboard_df.copy()
         hidden_leader_board_df = soap_original_df
-    elif PRIVATE_REPO and subset == "closed-ended-arabic":
-        leaderboard_table_df = closed_ended_arabic_leaderboard_df.copy()
-        hidden_leader_board_df = closed_ended_arabic_original_df
+    elif subset == "open_ended_arabic": 
+        leaderboard_table_df = open_ended_arabic_df.copy()
+        hidden_leader_board_df = open_ended_arabic_df
+    elif subset == "open_ended_french":
+        leaderboard_table_df = open_ended_french_df.copy()
+        hidden_leader_board_df = open_ended_french_df
+    elif subset == "open_ended_portuguese":
+        leaderboard_table_df = open_ended_portuguese_df.copy()
+        hidden_leader_board_df = open_ended_portuguese_df
+    elif subset == "open_ended_romanian":
+        leaderboard_table_df = open_ended_romanian_df.copy()
+        hidden_leader_board_df = open_ended_romanian_df
+    elif subset == "open_ended_greek":
+        leaderboard_table_df = open_ended_greek_df.copy()
+        hidden_leader_board_df = open_ended_greek_df
+    elif subset == "open_ended_spanish":
+        leaderboard_table_df = open_ended_spanish_df.copy()
+        hidden_leader_board_df = open_ended_spanish_df
+    elif subset == "closed_ended_multilingual":
+        leaderboard_table_df = closed_ended_multilingual_df.copy()
+        hidden_leader_board_df = closed_ended_multilingual_df
+    
     # else:
     #     match evaluation_metric:
     #         case "Span Based":
@@ -258,128 +312,140 @@ def filter_models(
 demo = gr.Blocks(css=custom_css)
 with demo:
     print("hello")
-    if PRIVATE_REPO:
-        gr.HTML(TITLE)  
     gr.HTML(LOGO)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    
+    
+    
 
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
+
+with gr.Blocks() as demo:
+    with gr.Tabs(elem_classes="tab-buttons") as outer_tabs:
+        with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=11):
+            with gr.Tabs(elem_classes="tab-buttons6") as language_tabs:
+                LANGUAGES = {
+                    "🇺🇸 English": "open_ended",
+                    "🇦🇪 Arabic": "open_ended_arabic",
+                    "🇫🇷 French": "open_ended_french",
+                    "🇪🇸 Spanish": "open_ended_spanish",
+                    "🇵🇹 Portuguese": "open_ended_portuguese",
+                    "🇷🇴 Romanian": "open_ended_romanian",
+                    "🇬🇷 Greek": "open_ended_greek",
+                }
+
+                for idx, (label, subset) in enumerate(LANGUAGES.items()):
+                    with gr.TabItem(label, elem_id=f"llm-benchmark-tab-open-{subset}", id=idx):
+                        # Custom judge information for each language
+                        if label == "🇺🇸 English":
+                            judge_text = "**Note:** Llama 3.1 70B Instruct has been used as judge for English."
+                        else:
+                            judge_text = "**Note:** Qwen 2.5 72B Instruct has been used as judge for this language."
+                        
+                        gr.Markdown(judge_text, elem_classes="markdown-text")
+                        
+                        with gr.Row():
+                            with gr.Column():
+                                with gr.Row():
+                                    search_bar = gr.Textbox(
+                                        placeholder=f"🔍 Search for your model in {label}...",
+                                        show_label=False,
+                                        elem_id=f"search-bar-{subset}",
+                                    )
+                                with gr.Row():
+                                    shown_columns = gr.CheckboxGroup(
+                                        choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
+                                        value=[
+                                            c.name
+                                            for c in fields(AutoEvalColumn)
+                                            if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)
+                                        ],
+                                        label="Select columns to show",
+                                        elem_id=f"column-select-{subset}",
+                                        interactive=True,
+                                    )
+                            with gr.Column(min_width=320):
+                                filter_columns_type = gr.CheckboxGroup(
+                                    label="Model Types",
+                                    choices=[t.to_str() for t in ModelType],
+                                    value=[t.to_str() for t in ModelType],
+                                    interactive=True,
+                                    elem_id=f"filter-columns-type-{subset}",
+                                )
+                                filter_domain_specific = gr.CheckboxGroup(
+                                    label="Domain Specificity",
+                                    choices=["🏥  Clinical models", "Generic models"],
+                                    value=["🏥  Clinical models", "Generic models"],
+                                    interactive=True,
+                                    elem_id=f"filter-columns-domain-{subset}",
+                                )
+                                filter_columns_size = gr.CheckboxGroup(
+                                    label="Model sizes (in billions of parameters)",
+                                    choices=list(NUMERIC_INTERVALS.keys()),
+                                    value=list(NUMERIC_INTERVALS.keys()),
+                                    interactive=True,
+                                    elem_id=f"filter-columns-size-{subset}",
+                                )
+
+                        datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset=subset)
+
+                        leaderboard_table = gr.Dataframe(
+                            value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                            headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                            datatype=TYPES,
+                            elem_id=f"leaderboard-table-{subset}",
+                            interactive=False,
+                            visible=True,
                         )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)
+
+                        hidden_leaderboard_table_for_search = gr.Dataframe(
+                            value=datasets_original_df[OPEN_ENDED_COLS],
+                            headers=OPEN_ENDED_COLS,
+                            datatype=TYPES,
+                            visible=False,
+                        )
+
+                        search_bar.submit(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
                             ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
+                            leaderboard_table,
                         )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model Types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
-                    filter_domain_specific = gr.CheckboxGroup(
-                        label="Domain Specificity",
-                        choices=["🏥  Clinical models", "Generic models"],
-                        value=["🏥  Clinical models", "Generic models"],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
 
-            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="open_ended")
+                        for selector in [
+                            shown_columns,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size,
+                        ]:
+                            selector.change(
+                                update_table,
+                                [
+                                    hidden_leaderboard_table_for_search,
+                                    shown_columns,
+                                    search_bar,
+                                    filter_columns_type,
+                                    filter_domain_specific,
+                                    filter_columns_size
+                                ],
+                                leaderboard_table,
+                                queue=True,
+                            )
+
+                        with gr.Accordion("💬 Generation templates", open=False):
+                            with gr.Accordion("Response generation", open=False):
+                                render_generation_templates(task="open_ended", generation_type="response_generation")
+                            with gr.Accordion("Scoring Rubric", open=False):
+                                render_generation_templates(task="open_ended", generation_type="scoring_rubric")
+
 
-            leaderboard_table = gr.components.Dataframe(
-                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
 
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=datasets_original_df[OPEN_ENDED_COLS],
-                headers=OPEN_ENDED_COLS,
-                datatype=TYPES,
-                visible=False,
-            )
 
-                        
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    search_bar,
-                    filter_columns_type,
-                    filter_domain_specific,
-                    filter_columns_size
-                    # filter_columns_architecture
-                ],
-                leaderboard_table,
-            )
-            for selector in [
-                shown_columns,
-                filter_columns_type,
-                filter_domain_specific,
-                # filter_columns_architecture,
-                filter_columns_size,
-                # deleted_models_visibility,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        search_bar,
-                        filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size
-                        # filter_columns_architecture,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-            with gr.Accordion("💬 Generation templates", open=False):
-                with gr.Accordion("Response generation", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="response_generation")
-                with gr.Accordion("Scoring Rubric", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="scoring_rubric")
         with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=2):
             with gr.Row():
                 with gr.Column():
@@ -387,7 +453,7 @@ with demo:
                         search_bar = gr.Textbox(
                             placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                             show_label=False,
-                            elem_id="search-bar",
+                            elem_id="search-bar-med-safety",
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
@@ -398,64 +464,50 @@ with demo:
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)
                             ],
                             label="Select columns to show",
-                            elem_id="column-select",
+                            elem_id="column-select-med-safety",
                             interactive=True,
                         )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
                 with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model Types",
                         choices=[t.to_str() for t in ModelType],
                         value=[t.to_str() for t in ModelType],
                         interactive=True,
-                        elem_id="filter-columns-type",
+                        elem_id="filter-columns-type-med-safety",
                     )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
                     filter_domain_specific = gr.CheckboxGroup(
                         label="Domain Specificity",
                         choices=["🏥  Clinical models", "Generic models"],
                         value=["🏥  Clinical models", "Generic models"],
                         interactive=True,
-                        elem_id="filter-columns-type",
+                        elem_id="filter-domain-specific-med-safety",
                     )
                     filter_columns_size = gr.CheckboxGroup(
                         label="Model sizes (in billions of parameters)",
                         choices=list(NUMERIC_INTERVALS.keys()),
                         value=list(NUMERIC_INTERVALS.keys()),
                         interactive=True,
-                        elem_id="filter-columns-size",
+                        elem_id="filter-columns-size-med-safety",
                     )
 
             datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="med_safety")
 
-            leaderboard_table = gr.components.Dataframe(
+            leaderboard_table = gr.Dataframe(
                 value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
-                elem_id="leaderboard-table",
+                elem_id="leaderboard-table-med-safety",
                 interactive=False,
                 visible=True,
             )
 
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+            hidden_leaderboard_table_for_search = gr.Dataframe(
                 value=datasets_original_df[MED_SAFETY_COLS],
                 headers=MED_SAFETY_COLS,
                 datatype=TYPES,
                 visible=False,
             )
-
-                        
+                    
             search_bar.submit(
                 update_table,
                 [
@@ -465,16 +517,15 @@ with demo:
                     filter_columns_type,
                     filter_domain_specific,
                     filter_columns_size
-                    # filter_columns_architecture
                 ],
                 leaderboard_table,
             )
+            
             for selector in [
                 shown_columns,
                 filter_columns_type,
                 filter_domain_specific,
                 filter_columns_size,
-                # deleted_models_visibility,
             ]:
                 selector.change(
                     update_table,
@@ -489,11 +540,13 @@ with demo:
                     leaderboard_table,
                     queue=True,
                 )
+                
             with gr.Accordion("💬 Generation templates", open=False):
                 with gr.Accordion("Response generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="response_generation")
                 with gr.Accordion("Scoring Rubric", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="scoring_rubric")
+                    
         with gr.TabItem("🏅 Medical Summarization", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(CROSS_EVALUATION_METRICS, elem_classes="markdown-text")
             with gr.Row():
@@ -502,7 +555,7 @@ with demo:
                         search_bar = gr.Textbox(
                             placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                             show_label=False,
-                            elem_id="search-bar",
+                            elem_id="search-bar-med-summarization",
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
@@ -513,64 +566,50 @@ with demo:
                                 if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
                             ],
                             label="Select columns to show",
-                            elem_id="column-select",
+                            elem_id="column-select-med-summarization",
                             interactive=True,
                         )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
                 with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model Types",
                         choices=[t.to_str() for t in ModelType],
                         value=[t.to_str() for t in ModelType],
                         interactive=True,
-                        elem_id="filter-columns-type",
+                        elem_id="filter-columns-type-med-summarization",
                     )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
                     filter_domain_specific = gr.CheckboxGroup(
                         label="Domain Specificity",
                         choices=["🏥  Clinical models", "Generic models"],
                         value=["🏥  Clinical models", "Generic models"],
                         interactive=True,
-                        elem_id="filter-columns-type",
+                        elem_id="filter-domain-specific-med-summarization",
                     )
                     filter_columns_size = gr.CheckboxGroup(
                         label="Model sizes (in billions of parameters)",
                         choices=list(NUMERIC_INTERVALS.keys()),
                         value=list(NUMERIC_INTERVALS.keys()),
                         interactive=True,
-                        elem_id="filter-columns-size",
+                        elem_id="filter-columns-size-med-summarization",
                     )
 
             datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="medical_summarization")
 
-            leaderboard_table = gr.components.Dataframe(
+            leaderboard_table = gr.Dataframe(
                 value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
                 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
-                elem_id="leaderboard-table",
+                elem_id="leaderboard-table-med-summarization",
                 interactive=False,
                 visible=True,
             )
 
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+            hidden_leaderboard_table_for_search = gr.Dataframe(
                 value=datasets_original_df[MEDICAL_SUMMARIZATION_COLS],
                 headers=MEDICAL_SUMMARIZATION_COLS,
                 datatype=TYPES,
                 visible=False,
             )
-
-                        
+                    
             search_bar.submit(
                 update_table,
                 [
@@ -580,16 +619,15 @@ with demo:
                     filter_columns_type,
                     filter_domain_specific,
                     filter_columns_size
-                    # filter_columns_architecture
                 ],
                 leaderboard_table,
             )
+            
             for selector in [
                 shown_columns,
                 filter_columns_type,
                 filter_domain_specific,
                 filter_columns_size,
-                # deleted_models_visibility,
             ]:
                 selector.change(
                     update_table,
@@ -604,24 +642,26 @@ with demo:
                     leaderboard_table,
                     queue=True,
                 )
+                
             with gr.Accordion("💬 Generation templates", open=False):
                 with gr.Accordion("Response generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="medical_summarization", generation_type="response_generation")
                 with gr.Accordion("Question generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
                 with gr.Accordion("Cross Examination", open=False):
-                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")  
+                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
+                    
         with gr.TabItem("🏅 Note generation", elem_id="llm-benchmark-tab-table", id=4):
             gr.Markdown(NOTE_GENERATION_METRICS, elem_classes="markdown-text")
-            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
-                with gr.TabItem("ACI Bench", elem_id="llm-benchmark-tab-table2", id=0):
+            with gr.Tabs(elem_classes="tab-buttons2") as note_tabs:
+                with gr.TabItem("ACI Bench", elem_id="llm-benchmark-tab-aci", id=0):
                     with gr.Row():
                         with gr.Column():
                             with gr.Row():
                                 search_bar = gr.Textbox(
                                     placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                                     show_label=False,
-                                    elem_id="search-bar",
+                                    elem_id="search-bar-aci",
                                 )
                             with gr.Row():
                                 shown_columns = gr.CheckboxGroup(
@@ -632,64 +672,50 @@ with demo:
                                         if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)
                                     ],
                                     label="Select columns to show",
-                                    elem_id="column-select",
+                                    elem_id="column-select-aci",
                                     interactive=True,
                                 )
-                            # with gr.Row():
-                            #     deleted_models_visibility = gr.Checkbox(
-                            #         value=False, label="Show gated/private/deleted models", interactive=True
-                            #     )
                         with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
                             filter_columns_type = gr.CheckboxGroup(
                                 label="Model Types",
                                 choices=[t.to_str() for t in ModelType],
                                 value=[t.to_str() for t in ModelType],
                                 interactive=True,
-                                elem_id="filter-columns-type",
+                                elem_id="filter-columns-type-aci",
                             )
-                            # filter_columns_architecture = gr.CheckboxGroup(
-                            #     label="Architecture Types",
-                            #     choices=[i.value.name for i in ModelArch],
-                            #     value=[i.value.name for i in ModelArch],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-architecture",
-                            # )
                             filter_domain_specific = gr.CheckboxGroup(
                                 label="Domain Specificity",
                                 choices=["🏥  Clinical models", "Generic models"],
                                 value=["🏥  Clinical models", "Generic models"],
                                 interactive=True,
-                                elem_id="filter-columns-type",
+                                elem_id="filter-domain-specific-aci",
                             )
                             filter_columns_size = gr.CheckboxGroup(
                                 label="Model sizes (in billions of parameters)",
                                 choices=list(NUMERIC_INTERVALS.keys()),
                                 value=list(NUMERIC_INTERVALS.keys()),
                                 interactive=True,
-                                elem_id="filter-columns-size",
+                                elem_id="filter-columns-size-aci",
                             )
 
                     datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="aci")
 
-                    leaderboard_table = gr.components.Dataframe(
+                    leaderboard_table = gr.Dataframe(
                         value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
                         headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                         datatype=TYPES,
-                        elem_id="leaderboard-table",
+                        elem_id="leaderboard-table-aci",
                         interactive=False,
                         visible=True,
                     )
 
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                    hidden_leaderboard_table_for_search = gr.Dataframe(
                         value=datasets_original_df[ACI_COLS],
                         headers=ACI_COLS,
                         datatype=TYPES,
                         visible=False,
                     )
-
-                                
+                            
                     search_bar.submit(
                         update_table,
                         [
@@ -699,16 +725,15 @@ with demo:
                             filter_columns_type,
                             filter_domain_specific,
                             filter_columns_size
-                            # filter_columns_architecture
                         ],
                         leaderboard_table,
                     )
+                    
                     for selector in [
                         shown_columns,
                         filter_columns_type,
                         filter_domain_specific,
                         filter_columns_size,
-                        # deleted_models_visibility,
                     ]:
                         selector.change(
                             update_table,
@@ -723,14 +748,15 @@ with demo:
                             leaderboard_table,
                             queue=True,
                         )
-                with gr.TabItem("SOAP Notes", elem_id="llm-benchmark-tab-table2", id=1):
+                
+                with gr.TabItem("SOAP Notes", elem_id="llm-benchmark-tab-soap", id=1):
                     with gr.Row():
                         with gr.Column():
                             with gr.Row():
                                 search_bar = gr.Textbox(
                                     placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
                                     show_label=False,
-                                    elem_id="search-bar",
+                                    elem_id="search-bar-soap",
                                 )
                             with gr.Row():
                                 shown_columns = gr.CheckboxGroup(
@@ -741,64 +767,50 @@ with demo:
                                         if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)
                                     ],
                                     label="Select columns to show",
-                                    elem_id="column-select",
+                                    elem_id="column-select-soap",
                                     interactive=True,
                                 )
-                            # with gr.Row():
-                            #     deleted_models_visibility = gr.Checkbox(
-                            #         value=False, label="Show gated/private/deleted models", interactive=True
-                            #     )
                         with gr.Column(min_width=320):
-                            # with gr.Box(elem_id="box-filter"):
                             filter_columns_type = gr.CheckboxGroup(
                                 label="Model Types",
                                 choices=[t.to_str() for t in ModelType],
                                 value=[t.to_str() for t in ModelType],
                                 interactive=True,
-                                elem_id="filter-columns-type",
+                                elem_id="filter-columns-type-soap",
                             )
-                            # filter_columns_architecture = gr.CheckboxGroup(
-                            #     label="Architecture Types",
-                            #     choices=[i.value.name for i in ModelArch],
-                            #     value=[i.value.name for i in ModelArch],
-                            #     interactive=True,
-                            #     elem_id="filter-columns-architecture",
-                            # )
                             filter_domain_specific = gr.CheckboxGroup(
                                 label="Domain Specificity",
                                 choices=["🏥  Clinical models", "Generic models"],
                                 value=["🏥  Clinical models", "Generic models"],
                                 interactive=True,
-                                elem_id="filter-columns-type",
+                                elem_id="filter-domain-specific-soap",
                             )
                             filter_columns_size = gr.CheckboxGroup(
                                 label="Model sizes (in billions of parameters)",
                                 choices=list(NUMERIC_INTERVALS.keys()),
                                 value=list(NUMERIC_INTERVALS.keys()),
                                 interactive=True,
-                                elem_id="filter-columns-size",
+                                elem_id="filter-columns-size-soap",
                             )
 
                     datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="soap")
 
-                    leaderboard_table = gr.components.Dataframe(
+                    leaderboard_table = gr.Dataframe(
                         value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
                         headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                         datatype=TYPES,
-                        elem_id="leaderboard-table",
+                        elem_id="leaderboard-table-soap",
                         interactive=False,
                         visible=True,
                     )
 
-                    # Dummy leaderboard for handling the case when the user uses backspace key
-                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                    hidden_leaderboard_table_for_search = gr.Dataframe(
                         value=datasets_original_df[SOAP_COLS],
                         headers=SOAP_COLS,
                         datatype=TYPES,
                         visible=False,
                     )
-
-                                
+                            
                     search_bar.submit(
                         update_table,
                         [
@@ -808,16 +820,15 @@ with demo:
                             filter_columns_type,
                             filter_domain_specific,
                             filter_columns_size
-                            # filter_columns_architecture
                         ],
                         leaderboard_table,
                     )
+                    
                     for selector in [
                         shown_columns,
                         filter_columns_type,
                         filter_domain_specific,
                         filter_columns_size,
-                        # deleted_models_visibility,
                     ]:
                         selector.change(
                             update_table,
@@ -832,6 +843,7 @@ with demo:
                             leaderboard_table,
                             queue=True,
                         )
+                        
             with gr.Accordion("💬 Generation templates", open=False):
                 with gr.Accordion("ACI-Bench Response generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="aci", generation_type="response_generation")
@@ -840,87 +852,93 @@ with demo:
                 with gr.Accordion("Question generation", open=False):
                     system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
                 with gr.Accordion("Cross Examination", open=False):
-                        system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
-        if PRIVATE_REPO:
-            with gr.TabItem("Dev Evals", elem_id="llm-benchmark-tab-table", id=100):
-                with gr.Tabs(elem_classes="tab-buttons2") as tabs:
-                    with gr.TabItem("🏅 Arabic Closed Ended Evaluation", elem_id="llm-benchmark-tab-table100", id=0):
-                        with gr.Row():
-                            with gr.Column():
-                                with gr.Row():
-                                    search_bar = gr.Textbox(
-                                        placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                                        show_label=False,
-                                        elem_id="search-bar",
-                                    )
-                                with gr.Row():
-                                    shown_columns = gr.CheckboxGroup(
-                                        choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_arabic_col)],
-                                        value=[
-                                            c.name
-                                            for c in fields(AutoEvalColumn)
-                                            if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_arabic_col)
-                                        ],
-                                        label="Select columns to show",
-                                        elem_id="column-select",
-                                        interactive=True,
-                                    )
-                                # with gr.Row():
-                                #     deleted_models_visibility = gr.Checkbox(
-                                #         value=False, label="Show gated/private/deleted models", interactive=True
-                                #     )
-                            with gr.Column(min_width=320):
-                                # with gr.Box(elem_id="box-filter"):
-                                filter_columns_type = gr.CheckboxGroup(
-                                    label="Model Types",
-                                    choices=[t.to_str() for t in ModelType],
-                                    value=[t.to_str() for t in ModelType],
-                                    interactive=True,
-                                    elem_id="filter-columns-type",
-                                )
-                                # filter_columns_architecture = gr.CheckboxGroup(
-                                #     label="Architecture Types",
-                                #     choices=[i.value.name for i in ModelArch],
-                                #     value=[i.value.name for i in ModelArch],
-                                #     interactive=True,
-                                #     elem_id="filter-columns-architecture",
-                                # )
-                                filter_domain_specific = gr.CheckboxGroup(
-                                    label="Domain Specificity",
-                                    choices=["🏥  Clinical models", "Generic models"],
-                                    value=["🏥  Clinical models", "Generic models"],
-                                    interactive=True,
-                                    elem_id="filter-columns-type",
+                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
+                    
+        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-closed", id=6):
+            with gr.Tabs(elem_classes="tab-buttons2") as closed_tabs:
+                # ENGLISH TAB
+                with gr.TabItem("English", elem_id="llm-benchmark-tab-closed-english", id=0):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar-closed-english",
                                 )
-                                filter_columns_size = gr.CheckboxGroup(
-                                    label="Model sizes (in billions of parameters)",
-                                    choices=list(NUMERIC_INTERVALS.keys()),
-                                    value=list(NUMERIC_INTERVALS.keys()),
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select-closed-english",
                                     interactive=True,
-                                    elem_id="filter-columns-size",
                                 )
+                        with gr.Column(min_width=320):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type-closed-english",
+                            )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-domain-specific-closed-english",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size-closed-english",
+                            )
 
-                        closed_ended_arabic_leaderboard_df, closed_ended_arabic_original_df = update_df(shown_columns.value, subset="closed-ended-arabic")
-
-                        leaderboard_table = gr.components.Dataframe(
-                            value=closed_ended_arabic_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                            headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                            datatype=TYPES,
-                            elem_id="leaderboard-table",
-                            interactive=False,
-                            visible=True,
-                        )
-
-                        # Dummy leaderboard for handling the case when the user uses backspace key
-                        hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                            value=closed_ended_arabic_original_df[CLOSED_ENDED_ARABIC_COLS],
-                            headers=CLOSED_ENDED_ARABIC_COLS,
-                            datatype=TYPES,
-                            visible=False,
-                        )
-
-                                    
-                        search_bar.submit(
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table-english",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[DATASET_COLS],
+                        headers=DATASET_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                        ],
+                        leaderboard_table,
+                    )
+                    
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                    ]:
+                        selector.change(
                             update_table,
                             [
                                 hidden_leaderboard_table_for_search,
@@ -929,256 +947,114 @@ with demo:
                                 filter_columns_type,
                                 filter_domain_specific,
                                 filter_columns_size
-                                # filter_columns_architecture
                             ],
                             leaderboard_table,
+                            queue=True,
                         )
-                        for selector in [
-                            shown_columns,
-                            filter_columns_type,
-                            filter_domain_specific,
-                            # filter_columns_architecture,
-                            filter_columns_size,
-                            # deleted_models_visibility,
-                        ]:
-                            selector.change(
-                                update_table,
-                                [
-                                    hidden_leaderboard_table_for_search,
-                                    shown_columns,
-                                    search_bar,
-                                    filter_columns_type,
-                                    filter_domain_specific,
-                                    filter_columns_size
-                                    # filter_columns_architecture,
-                                ],
-                                leaderboard_table,
-                                queue=True,
-                            )
-        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
+                
+                #MULTILINGUAL TAB - Same level as English tab
+                with gr.TabItem("🌍 Multilingual", elem_id="llm-benchmark-tab-table9", id=1):
                     with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
+                        gr.Markdown("📊 **Dataset Information:** This tab uses the Global MMLU dataset filtering only the subcategory: medical (10.7%)")
+                    
                     with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
-                    # with gr.Row():
-                    #     deleted_models_visibility = gr.Checkbox(
-                    #         value=False, label="Show gated/private/deleted models", interactive=True
-                    #     )
-                with gr.Column(min_width=320):
-                    # with gr.Box(elem_id="box-filter"):
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model Types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    # filter_columns_architecture = gr.CheckboxGroup(
-                    #     label="Architecture Types",
-                    #     choices=[i.value.name for i in ModelArch],
-                    #     value=[i.value.name for i in ModelArch],
-                    #     interactive=True,
-                    #     elem_id="filter-columns-architecture",
-                    # )
-                    filter_domain_specific = gr.CheckboxGroup(
-                        label="Domain Specificity",
-                        choices=["🏥  Clinical models", "Generic models"],
-                        value=["🏥  Clinical models", "Generic models"],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Model sizes (in billions of parameters)",
-                        choices=list(NUMERIC_INTERVALS.keys()),
-                        value=list(NUMERIC_INTERVALS.keys()),
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
-
-            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
-
-            leaderboard_table = gr.components.Dataframe(
-                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=datasets_original_df[DATASET_COLS],
-                headers=DATASET_COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-                        
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    search_bar,
-                    filter_columns_type,
-                    filter_domain_specific,
-                    filter_columns_size
-                    # filter_columns_architecture
-                ],
-                leaderboard_table,
-            )
-            for selector in [
-                shown_columns,
-                filter_columns_type,
-                filter_domain_specific,
-                # filter_columns_architecture,
-                filter_columns_size,
-                # deleted_models_visibility,
-            ]:
-                selector.change(
-                    update_table,
-                    [
-                        hidden_leaderboard_table_for_search,
-                        shown_columns,
-                        search_bar,
-                        filter_columns_type,
-                        filter_domain_specific,
-                        filter_columns_size
-                        # filter_columns_architecture,
-                    ],
-                    leaderboard_table,
-                    queue=True,
-                )
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=5):
-            gr.Markdown(LLM_BENCHMARKS_TEXT_1, elem_classes="markdown-text")
-            gr.HTML(FIVE_PILLAR_DIAGRAM)
-            gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
-            # gr.HTML(EVALUATION_EXAMPLE_IMG, elem_classes="logo")
-            # gr.Markdown(LLM_BENCHMARKS_TEXT_2, elem_classes="markdown-text")
-            # gr.HTML(ENTITY_DISTRIBUTION_IMG, elem_classes="logo")
-            # gr.Markdown(LLM_BENCHMARKS_TEXT_3, elem_classes="markdown-text")
-            
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=6):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                                
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
                             )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
                             )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
                             )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")                    
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="closed_ended_multilingual")
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
                     )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="auto",
-                        interactive=True,
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[ClosedEndedMultilingual_COLS],
+                        headers=ClosedEndedMultilingual_COLS,
+                        datatype=TYPES,
+                        visible=False,
                     )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value=WeightType.Original.value.name,
-                        interactive=False,
+                        
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)", interactive=False)
-            with gr.Row():
-                domain_specific_toggle = gr.Checkbox(
-                    label="Domain specific", 
-                    value=False,
-                    info="Is your model medically oriented?",
-                )
-                chat_template_toggle = gr.Checkbox(
-                    label="Use chat template", 
-                    value=False,
-                    info="Is your model a chat model?",
-                )
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    model_type,
-                    domain_specific_toggle,
-                    chat_template_toggle,
-                    precision,
-                    weight_type
-                ],
-                submission_result,
-            )
-
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        # filter_columns_architecture,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                                # filter_columns_architecture,
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+    with gr.Row():     
+        with gr.Accordion("📙 Citation", open=False):      
             citation_button = gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,
@@ -1190,4 +1066,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch(allowed_paths=['./assets/'])
+demo.queue(default_concurrency_limit=40).launch(allowed_paths=['./assets/']) 

app_original.py

@@ -0,0 +1,1276 @@
+import subprocess
+
+import gradio as gr
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT_1,
+    LLM_BENCHMARKS_TEXT_2,
+    CROSS_EVALUATION_METRICS,
+    NOTE_GENERATION_METRICS,
+    # EVALUATION_EXAMPLE_IMG,
+    # LLM_BENCHMARKS_TEXT_2,
+    # ENTITY_DISTRIBUTION_IMG,
+    # LLM_BENCHMARKS_TEXT_3,
+    TITLE,
+    LOGO,
+    FIVE_PILLAR_DIAGRAM
+)
+from src.display.css_html_js import custom_css
+# changes to be made here
+from src.display.utils import (
+    DATASET_BENCHMARK_COLS,
+    OPEN_ENDED_BENCHMARK_COLS,
+    MED_SAFETY_BENCHMARK_COLS,
+    MEDICAL_SUMMARIZATION_BENCHMARK_COLS,
+    ACI_BENCHMARK_COLS,
+    SOAP_BENCHMARK_COLS,
+    #CLOSED_ENDED_ARABIC_BENCHMARK_COLS,
+    DATASET_COLS,
+    OPEN_ENDED_COLS,
+    MED_SAFETY_COLS,
+    MEDICAL_SUMMARIZATION_COLS,
+    ACI_COLS,
+    SOAP_COLS,
+    #CLOSED_ENDED_ARABIC_COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    NUMERIC_INTERVALS,
+    TYPES,
+    AutoEvalColumn,
+    ModelType,
+    ModelArch,
+    PromptTemplateName,
+    Precision,
+    WeightType,
+    fields,
+    render_generation_templates,
+    OpenEndedArabic_COLS,
+    OpenEndedArabic_BENCHMARK_COLS,
+    OpenEndedFrench_COLS,
+    OpenEndedFrench_BENCHMARK_COLS,
+    OpenEndedPortuguese_COLS,
+    OpenEndedPortuguese_BENCHMARK_COLS,
+    OpenEndedRomanian_COLS,
+    OpenEndedRomanian_BENCHMARK_COLS,
+    OpenEndedGreek_COLS,
+    OpenEndedGreek_BENCHMARK_COLS,
+    OpenEndedSpanish_COLS,
+    OpenEndedSpanish_BENCHMARK_COLS,
+    ClosedEndedMultilingual_COLS,
+    ClosedEndedMultilingual_BENCHMARK_COLS,
+    
+    #closed_ended_multilingual,
+    # Open_EndedArabic,
+    # Open_EndedSpanish,
+    # Open_EndedFrench,
+    # Open_EndedPortuguese,
+    # Open_EndedRomanian,
+    # Open_EndedGreek,
+    # Open_EndedSpanish,
+    # Open_EndedArabic,
+    # Open_EndedFrench,
+    
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN, PRIVATE_REPO
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval, PLACEHOLDER_DATASET_WISE_NORMALIZATION_CONFIG
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+# Span based results
+# changes to be made here
+
+_, harness_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "accuracy", "datasets")
+harness_datasets_leaderboard_df = harness_datasets_original_df.copy()
+
+_, open_ended_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OPEN_ENDED_COLS, OPEN_ENDED_BENCHMARK_COLS, "score", "open_ended")
+open_ended_leaderboard_df = open_ended_original_df.copy()
+
+_, med_safety_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MED_SAFETY_COLS, MED_SAFETY_BENCHMARK_COLS, "score", "med_safety")
+med_safety_leaderboard_df = med_safety_original_df.copy()
+
+_, medical_summarization_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, MEDICAL_SUMMARIZATION_COLS, MEDICAL_SUMMARIZATION_BENCHMARK_COLS, "score", "medical_summarization")
+medical_summarization_leaderboard_df = medical_summarization_original_df.copy()
+
+_, aci_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ACI_COLS, ACI_BENCHMARK_COLS, "score", "aci")
+aci_leaderboard_df = aci_original_df.copy()
+
+_, soap_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, SOAP_COLS, SOAP_BENCHMARK_COLS, "score", "soap")
+soap_leaderboard_df = soap_original_df.copy()
+
+
+_, open_ended_arabic_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedArabic_COLS, OpenEndedArabic_BENCHMARK_COLS, "score", "open_ended_arabic")
+_, open_ended_french_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedFrench_COLS, OpenEndedFrench_BENCHMARK_COLS, "score", "open_ended_french")
+_, open_ended_portuguese_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedPortuguese_COLS, OpenEndedPortuguese_BENCHMARK_COLS, "score", "open_ended_portuguese")
+_, open_ended_romanian_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedRomanian_COLS, OpenEndedRomanian_BENCHMARK_COLS, "score", "open_ended_romanian")
+_, open_ended_greek_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedGreek_COLS, OpenEndedGreek_BENCHMARK_COLS, "score", "open_ended_greek")
+_, open_ended_spanish_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, OpenEndedSpanish_COLS, OpenEndedSpanish_BENCHMARK_COLS, "score", "open_ended_spanish")
+_, closed_ended_multilingual_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ClosedEndedMultilingual_COLS, ClosedEndedMultilingual_BENCHMARK_COLS, "score", "closed_ended_multilingual")
+
+
+open_ended_arabic_leaderboard_df = open_ended_arabic_df.copy()
+open_ended_french_leaderboard_df = open_ended_french_df.copy()
+open_ended_portuguese_leaderboard_df = open_ended_portuguese_df.copy()
+open_ended_romanian_leaderboard_df = open_ended_romanian_df.copy()
+open_ended_greek_leaderboard_df = open_ended_greek_df.copy()
+open_ended_spanish_leaderboard_df = open_ended_spanish_df.copy()
+closed_ended_multilingual_leaderboard_df = closed_ended_multilingual_df.copy()
+
+
+# if PRIVATE_REPO:
+#     _, closed_ended_arabic_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, CLOSED_ENDED_ARABIC_COLS, CLOSED_ENDED_ARABIC_BENCHMARK_COLS, "score", "closed_ended_arabic")
+#     closed_ended_arabic_leaderboard_df = closed_ended_arabic_original_df.copy()
+
+# breakpoint()
+# # Token based results
+# _, token_based_datasets_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, DATASET_COLS, DATASET_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "datasets")
+# token_based_datasets_leaderboard_df = token_based_datasets_original_df.copy()
+
+# _, token_based_types_original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, Clinical_TYPES_COLS, TYPES_BENCHMARK_COLS, "TokenBasedWithMacroAverage", "clinical_types")
+# token_based_types_leaderboard_df = token_based_types_original_df.copy()
+
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+# breakpoint()
+def update_df(shown_columns, subset="datasets"):
+    # changes to be made here
+    if subset == "datasets":
+        leaderboard_table_df = harness_datasets_leaderboard_df.copy()
+        hidden_leader_board_df = harness_datasets_original_df
+    elif subset == "open_ended":
+        leaderboard_table_df = open_ended_leaderboard_df.copy()
+        hidden_leader_board_df = open_ended_original_df
+    elif subset == "med_safety":
+        leaderboard_table_df = med_safety_leaderboard_df.copy()
+        hidden_leader_board_df = med_safety_original_df
+    elif subset == "medical_summarization":
+        leaderboard_table_df = medical_summarization_leaderboard_df.copy()
+        hidden_leader_board_df = medical_summarization_original_df
+    elif subset == "aci":
+        leaderboard_table_df = aci_leaderboard_df.copy()
+        hidden_leader_board_df = aci_original_df
+    elif subset == "soap":
+        leaderboard_table_df = soap_leaderboard_df.copy()
+        hidden_leader_board_df = soap_original_df
+    elif subset == "open_ended_arabic": 
+        leaderboard_table_df = open_ended_arabic_df.copy()
+        hidden_leader_board_df = open_ended_arabic_df
+    elif subset == "open_ended_french":
+        leaderboard_table_df = open_ended_french_df.copy()
+        hidden_leader_board_df = open_ended_french_df
+    elif subset == "open_ended_portuguese":
+        leaderboard_table_df = open_ended_portuguese_df.copy()
+        hidden_leader_board_df = open_ended_portuguese_df
+    elif subset == "open_ended_romanian":
+        leaderboard_table_df = open_ended_romanian_df.copy()
+        hidden_leader_board_df = open_ended_romanian_df
+    elif subset == "open_ended_greek":
+        leaderboard_table_df = open_ended_greek_df.copy()
+        hidden_leader_board_df = open_ended_greek_df
+    elif subset == "open_ended_spanish":
+        leaderboard_table_df = open_ended_spanish_df.copy()
+        hidden_leader_board_df = open_ended_spanish_df
+    elif subset == "closed_ended_multilingual":
+        leaderboard_table_df = closed_ended_multilingual_df.copy()
+        hidden_leader_board_df = closed_ended_multilingual_df
+    
+    # else:
+    #     match evaluation_metric:
+    #         case "Span Based":
+    #             leaderboard_table_df = span_based_types_leaderboard_df.copy()
+    #             hidden_leader_board_df = span_based_types_original_df
+    #         case "Token Based":
+    #             leaderboard_table_df = token_based_types_leaderboard_df.copy()
+    #             hidden_leader_board_df = token_based_types_original_df
+    #         case _:
+    #             pass        
+    
+
+    value_cols = [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns
+    # breakpoint()
+    return leaderboard_table_df[value_cols], hidden_leader_board_df
+
+
+# Searching and filtering
+def update_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    query: str = "",
+    type_query: list = None,
+    domain_specific_query: list = None,
+    size_query: list = None,
+    precision_query: str = None,
+    show_deleted: bool = False,
+):
+    # breakpoint()
+    filtered_df = filter_models(hidden_df, type_query, domain_specific_query, size_query, precision_query, show_deleted)
+    # breakpoint()
+    filtered_df = filter_queries(query, filtered_df)
+    # breakpoint()
+    df = select_columns(filtered_df, columns, list(hidden_df.columns))
+    # breakpoint()
+    return df
+
+
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
+
+
+def select_columns(df: pd.DataFrame, columns: list, cols:list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[always_here_cols + [c for c in cols if c in df.columns and c in columns]]
+    return filtered_df
+
+
+def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
+    final_df = []
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[
+                    AutoEvalColumn.model.name,
+                    # AutoEvalColumn.precision.name,
+                    # AutoEvalColumn.revision.name,
+                ]
+            )
+
+    return filtered_df
+
+
+def filter_models(
+    df: pd.DataFrame, type_query: list, domain_specific_query: list, size_query: list, precision_query: list, show_deleted: bool
+) -> pd.DataFrame:
+    # Show all models
+    # if show_deleted:
+    #     filtered_df = df
+    # else:  # Show only still on the hub models
+    #     filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
+
+    filtered_df = df
+
+    if type_query is not None:
+        type_name = [t.split(" ")[1] for t in type_query]
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type.name].isin(type_name)]
+
+    if domain_specific_query is not None:
+        domain_specifics = []
+        if "🏥  Clinical models" in domain_specific_query:
+            domain_specifics.append(True)
+        if "Generic models" in domain_specific_query:
+            domain_specifics.append(False)
+        filtered_df = filtered_df.loc[df[AutoEvalColumn.is_domain_specific.name].isin(domain_specifics)]
+        
+    # if architecture_query is not None:
+    #     arch_types = [t for t in architecture_query]
+    #     filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(arch_types)]
+    #         # filtered_df = filtered_df.loc[df[AutoEvalColumn.architecture.name].isin(architecture_query + ["None"])]
+    
+    if precision_query is not None:
+        if AutoEvalColumn.precision.name in df.columns:
+            filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
+
+    if size_query is not None:
+        numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
+        params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+        mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+        filtered_df = filtered_df.loc[mask]
+
+    return filtered_df
+
+            
+demo = gr.Blocks(css=custom_css)
+with demo:
+    print("hello")
+    gr.HTML(LOGO)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+
+
+
+with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🌍 Open Ended MultilingualEvaluation", elem_id="llm-benchmark-tab-table", id=11):
+            with gr.Tabs(elem_classes="tab-buttons6") as tabs:
+                with gr.TabItem("English", elem_id="llm-benchmark-tab-table10", id=0):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                                with gr.Row():
+                                    shown_columns = gr.CheckboxGroup(
+                                        choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                            
+                            )
+                                    
+                                    with gr.Column(min_width=320):
+                                        filter_columns_type = gr.CheckboxGroup(
+                                            label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                            )
+                                        filter_domain_specific = gr.CheckboxGroup(
+                                            label="Domain Specificity",
+                                            choices=["🏥  Clinical models", "Generic models"],
+                                            value=["🏥  Clinical models", "Generic models"],
+                                            interactive=True,
+                                            elem_id="filter-columns-type",
+                    )
+                                        filter_domain_specific = gr.CheckboxGroup(
+                                            label="Domain Specificity",
+                                            choices=["🏥  Clinical models", "Generic models"],
+                                            value=["🏥  Clinical models", "Generic models"],
+                                            interactive=True,
+                                            elem_id="filter-columns-type",
+                    )
+                                        filter_columns_size = gr.CheckboxGroup(
+                                            label="Model sizes (in billions of parameters)",
+                                            choices=list(NUMERIC_INTERVALS.keys()),
+                                            value=list(NUMERIC_INTERVALS.keys()),
+                                            interactive=True,
+                                            elem_id="filter-columns-size",
+                    )
+                                        
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="open_ended")
+            
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[OPEN_ENDED_COLS],
+                headers=OPEN_ENDED_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                # filter_columns_architecture,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                        # filter_columns_architecture,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+             
+            
+            with gr.Accordion("💬 Generation templates", open=False):
+                with gr.Accordion("Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="response_generation")
+                with gr.Accordion("Scoring Rubric", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="scoring_rubric")       
+
+        
+        with gr.TabItem("🏅 Open Ended Evaluation", elem_id="llm-benchmark-tab-table", id=1):
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.open_ended_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain Specificity",
+                        choices=["🏥  Clinical models", "Generic models"],
+                        value=["🏥  Clinical models", "Generic models"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="open_ended")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[OPEN_ENDED_COLS],
+                headers=OPEN_ENDED_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                # filter_columns_architecture,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                        # filter_columns_architecture,
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+            with gr.Accordion("💬 Generation templates", open=False):
+                with gr.Accordion("Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="response_generation")
+                with gr.Accordion("Scoring Rubric", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="open_ended", generation_type="scoring_rubric")
+                    
+        with gr.TabItem("🏅 Med Safety", elem_id="llm-benchmark-tab-table", id=2):
+            with gr.Row():
+                with gr.Column():
+                    
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                        
+                        
+                        with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.med_safety_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                        
+                        
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                        with gr.Column(min_width=320):
+                            
+                    # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                            label="Model Types",
+                            choices=[t.to_str() for t in ModelType],
+                            value=[t.to_str() for t in ModelType],
+                            interactive=True,
+                            elem_id="filter-columns-type",
+                        )
+                            
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain Specificity",
+                        choices=["🏥  Clinical models", "Generic models"],
+                        value=["🏥  Clinical models", "Generic models"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                            )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="med_safety")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[MED_SAFETY_COLS],
+                headers=MED_SAFETY_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+            with gr.Accordion("💬 Generation templates", open=False):
+                with gr.Accordion("Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="response_generation")
+                with gr.Accordion("Scoring Rubric", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="med_safety", generation_type="scoring_rubric")
+        with gr.TabItem("🏅 Medical Summarization", elem_id="llm-benchmark-tab-table", id=3):
+            gr.Markdown(CROSS_EVALUATION_METRICS, elem_classes="markdown-text")
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)],
+                            value=[
+                                c.name
+                                for c in fields(AutoEvalColumn)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.medical_summarization_col)
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    # with gr.Row():
+                    #     deleted_models_visibility = gr.Checkbox(
+                    #         value=False, label="Show gated/private/deleted models", interactive=True
+                    #     )
+                with gr.Column(min_width=320):
+                    # with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model Types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    # filter_columns_architecture = gr.CheckboxGroup(
+                    #     label="Architecture Types",
+                    #     choices=[i.value.name for i in ModelArch],
+                    #     value=[i.value.name for i in ModelArch],
+                    #     interactive=True,
+                    #     elem_id="filter-columns-architecture",
+                    # )
+                    filter_domain_specific = gr.CheckboxGroup(
+                        label="Domain Specificity",
+                        choices=["🏥  Clinical models", "Generic models"],
+                        value=["🏥  Clinical models", "Generic models"],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+
+            datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="medical_summarization")
+
+            leaderboard_table = gr.components.Dataframe(
+                value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                datatype=TYPES,
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=datasets_original_df[MEDICAL_SUMMARIZATION_COLS],
+                headers=MEDICAL_SUMMARIZATION_COLS,
+                datatype=TYPES,
+                visible=False,
+            )
+
+                        
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    search_bar,
+                    filter_columns_type,
+                    filter_domain_specific,
+                    filter_columns_size
+                    # filter_columns_architecture
+                ],
+                leaderboard_table,
+            )
+            for selector in [
+                shown_columns,
+                filter_columns_type,
+                filter_domain_specific,
+                filter_columns_size,
+                # deleted_models_visibility,
+            ]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        search_bar,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+            with gr.Accordion("💬 Generation templates", open=False):
+                with gr.Accordion("Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="medical_summarization", generation_type="response_generation")
+                with gr.Accordion("Question generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
+                with gr.Accordion("Cross Examination", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")  
+        with gr.TabItem("🏅 Note generation", elem_id="llm-benchmark-tab-table", id=4):
+            gr.Markdown(NOTE_GENERATION_METRICS, elem_classes="markdown-text")
+            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
+                with gr.TabItem("ACI Bench", elem_id="llm-benchmark-tab-table2", id=0):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.aci_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="aci")
+
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[ACI_COLS],
+                        headers=ACI_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+                with gr.TabItem("SOAP Notes", elem_id="llm-benchmark-tab-table2", id=1):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.soap_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="soap")
+
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[SOAP_COLS],
+                        headers=SOAP_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+            with gr.Accordion("💬 Generation templates", open=False):
+                with gr.Accordion("ACI-Bench Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="aci", generation_type="response_generation")
+                with gr.Accordion("SOAP Notes Response generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="soap", generation_type="response_generation")
+                with gr.Accordion("Question generation", open=False):
+                    system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="question_generation")
+                with gr.Accordion("Cross Examination", open=False):
+                        system_prompt, user_prompt = render_generation_templates(task="ce", generation_type="cross_examination")
+        with gr.TabItem("🏅 Closed Ended Evaluation", elem_id="llm-benchmark-tab-table", id=6):
+            with gr.Tabs(elem_classes="tab-buttons2") as tabs:
+                with gr.TabItem("English", elem_id="llm-benchmark-tab-table9", id=0):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.dataset_task_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="datasets")
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[DATASET_COLS],
+                        headers=DATASET_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        # filter_columns_architecture,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                                # filter_columns_architecture,
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+                
+                with gr.TabItem("🌍 Multilingual", elem_id="llm-benchmark-tab-table9", id=1):
+                    with gr.Row():
+                        with gr.Column():
+                            with gr.Row():
+                                search_bar = gr.Textbox(
+                                    placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                                    show_label=False,
+                                    elem_id="search-bar",
+                                )
+                            with gr.Row():
+                                shown_columns = gr.CheckboxGroup(
+                                    choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)],
+                                    value=[
+                                        c.name
+                                        for c in fields(AutoEvalColumn)
+                                        if c.displayed_by_default and not c.hidden and not c.never_hidden and (c.invariant or c.closed_ended_multilingual_col)
+                                    ],
+                                    label="Select columns to show",
+                                    elem_id="column-select",
+                                    interactive=True,
+                                )
+                            # with gr.Row():
+                            #     deleted_models_visibility = gr.Checkbox(
+                            #         value=False, label="Show gated/private/deleted models", interactive=True
+                            #     )
+                        with gr.Column(min_width=320):
+                            # with gr.Box(elem_id="box-filter"):
+                            filter_columns_type = gr.CheckboxGroup(
+                                label="Model Types",
+                                choices=[t.to_str() for t in ModelType],
+                                value=[t.to_str() for t in ModelType],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            # filter_columns_architecture = gr.CheckboxGroup(
+                            #     label="Architecture Types",
+                            #     choices=[i.value.name for i in ModelArch],
+                            #     value=[i.value.name for i in ModelArch],
+                            #     interactive=True,
+                            #     elem_id="filter-columns-architecture",
+                            # )
+                            filter_domain_specific = gr.CheckboxGroup(
+                                label="Domain Specificity",
+                                choices=["🏥  Clinical models", "Generic models"],
+                                value=["🏥  Clinical models", "Generic models"],
+                                interactive=True,
+                                elem_id="filter-columns-type",
+                            )
+                            filter_columns_size = gr.CheckboxGroup(
+                                label="Model sizes (in billions of parameters)",
+                                choices=list(NUMERIC_INTERVALS.keys()),
+                                value=list(NUMERIC_INTERVALS.keys()),
+                                interactive=True,
+                                elem_id="filter-columns-size",
+                            )
+
+                    datasets_leaderboard_df, datasets_original_df = update_df(shown_columns.value, subset="closed_ended_multilingual")
+                    leaderboard_table = gr.components.Dataframe(
+                        value=datasets_leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value],
+                        headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                        datatype=TYPES,
+                        elem_id="leaderboard-table",
+                        interactive=False,
+                        visible=True,
+                    )
+
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                        value=datasets_original_df[ClosedEndedMultilingual_COLS],
+                        headers=ClosedEndedMultilingual_COLS,
+                        datatype=TYPES,
+                        visible=False,
+                    )
+                                
+                    search_bar.submit(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            search_bar,
+                            filter_columns_type,
+                            filter_domain_specific,
+                            filter_columns_size
+                            # filter_columns_architecture
+                        ],
+                        leaderboard_table,
+                    )
+                    for selector in [
+                        shown_columns,
+                        filter_columns_type,
+                        filter_domain_specific,
+                        # filter_columns_architecture,
+                        filter_columns_size,
+                        # deleted_models_visibility,
+                    ]:
+                        selector.change(
+                            update_table,
+                            [
+                                hidden_leaderboard_table_for_search,
+                                shown_columns,
+                                search_bar,
+                                filter_columns_type,
+                                filter_domain_specific,
+                                filter_columns_size
+                                # filter_columns_architecture,
+                            ],
+                            leaderboard_table,
+                            queue=True,
+                        )
+                
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch(allowed_paths=['./assets/'])

src/about.py

@@ -40,6 +40,77 @@ class OpenEndedColumns(Enum):
     column3 = OpenEndedColumn("Score_intervals", "score", "Score 95% CI")
 # changes to be made here
 
+
+@dataclass
+class OpenEndedMultilingualColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+    
+class OpenEndedArabicColumn(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    arabic_column0 = OpenEndedMultilingualColumn("ELO", "score", "ELO")
+    arabic_column1 = OpenEndedMultilingualColumn("ELO_intervals", "score", "ELO 95% CI")
+    arabic_column2 = OpenEndedMultilingualColumn("Score", "score", "Score") 
+    arabic_column3 = OpenEndedMultilingualColumn("Score_intervals", "score", "Score 95% CI")
+    
+
+class OpenEndedFrenchColumn(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    french_column0 = OpenEndedMultilingualColumn("ELO", "score", "ELO")
+    french_column1 = OpenEndedMultilingualColumn("ELO_intervals", "score", "ELO 95% CI")
+    french_column2 = OpenEndedMultilingualColumn("Score", "score", "Score") 
+    french_column3 = OpenEndedMultilingualColumn("Score_intervals", "score", "Score 95% CI")
+    
+
+class OpenEndedSpanishColumn(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    spanish_column0 = OpenEndedMultilingualColumn("ELO", "score", "ELO")
+    spanish_column1 = OpenEndedMultilingualColumn("ELO_intervals", "score", "ELO 95% CI")
+    spanish_column2 = OpenEndedMultilingualColumn("Score", "score", "Score") 
+    spanish_column3 = OpenEndedMultilingualColumn("Score_intervals", "score", "Score 95% CI")
+    
+
+class OpenEndedPortugueseColumn(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    porto_column0 = OpenEndedMultilingualColumn("ELO", "score", "ELO")
+    porto_column1 = OpenEndedMultilingualColumn("ELO_intervals", "score", "ELO 95% CI")
+    porto_column2 = OpenEndedMultilingualColumn("Score", "score", "Score") 
+    porto_column3 = OpenEndedMultilingualColumn("Score_intervals", "score", "Score 95% CI")
+    
+
+class OpenEndedRomanianColumn(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    rom_column0 = OpenEndedMultilingualColumn("ELO", "score", "ELO")
+    rom_column1 = OpenEndedMultilingualColumn("ELO_intervals", "score", "ELO 95% CI")
+    rom_column2 = OpenEndedMultilingualColumn("Score", "score", "Score") 
+    rom_column3 = OpenEndedMultilingualColumn("Score_intervals", "score", "Score 95% CI")
+    
+
+class OpenEndedGreekColumn(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    greek_column0 = OpenEndedMultilingualColumn("ELO", "score", "ELO")
+    greek_column1 = OpenEndedMultilingualColumn("ELO_intervals", "score", "ELO 95% CI")
+    greek_column2 = OpenEndedMultilingualColumn("Score", "score", "Score") 
+    greek_column3 = OpenEndedMultilingualColumn("Score_intervals", "score", "Score 95% CI")
+    
+    
+
+@dataclass
+class ClosedEndedMultilingualColumn:
+    benchmark: str
+    metric: str
+    col_name: str
+    
+    
+class ClosedEndedMultilingualColumns(Enum):
+    mtask0 = ClosedEndedMultilingualColumn("Global-MMLU-Arabic", "accuracy", "🇦🇪Arabic")
+    mtask1 = ClosedEndedMultilingualColumn("Global-MMLU-French", "accuracy", "🇫🇷French")
+    mtask2 = ClosedEndedMultilingualColumn("Global-MMLU-Spanish", "accuracy", "🇪🇸Spanish")
+    mtask3 = ClosedEndedMultilingualColumn("Global-MMLU-Portuguese", "accuracy", "🇵🇹Portuguese")
+    mtask4 = ClosedEndedMultilingualColumn("Global-MMLU-Romanian", "accuracy", "🇷🇴Romanian")
+    mtask5 = ClosedEndedMultilingualColumn("Global-MMLU-Greek", "accuracy", "🇬🇷Greek")
+
 @dataclass
 class MedSafetyColumn:
     benchmark: str
@@ -102,11 +173,16 @@ class ClosedEndedArabicColumn:
     metric: str
     col_name: str
 
-class ClosedEndedArabicColumns(Enum):
-    arabictask0 = ClosedEndedArabicColumn("MMLU-Arabic", "accuracy", "MMLU-Arabic")
-    arabictask2 = ClosedEndedArabicColumn("MedMCQA-Arabic", "accuracy", "MedMCQA-Arabic")
-    arabictask3 = ClosedEndedArabicColumn("MedQA-Arabic", "accuracy", "MedQA-Arabic")
-    arabictask5 = ClosedEndedArabicColumn("PubMedQA-Arabic", "accuracy", "PubMedQA-Arabic")
+
+    
+    
+    
+    
+# class ClosedEndedArabicColumns(Enum):
+#     arabictask0 = ClosedEndedArabicColumn("MMLU-Arabic", "accuracy", "MMLU-Arabic")
+#     arabictask2 = ClosedEndedArabicColumn("MedMCQA-Arabic", "accuracy", "MedMCQA-Arabic")
+#     arabictask3 = ClosedEndedArabicColumn("MedQA-Arabic", "accuracy", "MedQA-Arabic")
+#     arabictask5 = ClosedEndedArabicColumn("PubMedQA-Arabic", "accuracy", "PubMedQA-Arabic")
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/display/utils.py

@@ -4,7 +4,7 @@ from enum import Enum
 import pandas as pd
 
 # changes to be made here
-from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.about import HarnessTasks, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedMultilingualColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn
 from src.envs import PRIVATE_REPO
 import json
 import gradio as gr
@@ -31,17 +31,21 @@ class ColumnContent:
     medical_summarization_col: bool = False
     aci_col: bool = False
     soap_col: bool = False
-    closed_ended_arabic_col: bool = False
+    open_ended_arabic_col: bool = False
+    open_ended_french_col: bool = False
+    open_ended_spanish_col: bool = False
+    open_ended_portuguese_col: bool = False
+    open_ended_romanian_col: bool = False
+    open_ended_greek_col: bool = False
+    closed_ended_multilingual_col: bool = False
 
 
-## Leaderboard columns
-auto_eval_column_dict = []
 # Init
 auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, True)])
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, closed_ended_arabic_col=True, invariant=False)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average", "number", True, False, dataset_task_col=True, closed_ended_multilingual_col=True, invariant=False)])
 auto_eval_column_dict.append(["overall", ColumnContent, ColumnContent("Overall Score", "number", True, False, medical_summarization_col=True, aci_col=True, soap_col=True, invariant=False)])
 for task in HarnessTasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True, False, dataset_task_col=True, invariant=False)])
@@ -59,9 +63,21 @@ for column in ACIColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, aci_col=True, invariant=False)])
 for column in SOAPColumns:
     auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, soap_col=True, invariant=False)])
-# if PRIVATE_REPO:
-for column in ClosedEndedArabicColumns:
-    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, closed_ended_arabic_col=True, invariant=False)])
+for column in OpenEndedArabicColumn:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_arabic_col=True, invariant=False)])
+for column in OpenEndedFrenchColumn:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_french_col=True, invariant=False)])
+for column in OpenEndedSpanishColumn:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_spanish_col=True, invariant=False)])
+for column in OpenEndedPortugueseColumn:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_portuguese_col=True, invariant=False)])
+for column in OpenEndedRomanianColumn:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_romanian_col=True, invariant=False)])
+for column in OpenEndedGreekColumn:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, open_ended_greek_col=True, invariant=False)])
+for column in ClosedEndedMultilingualColumns:
+    auto_eval_column_dict.append([column.name, ColumnContent, ColumnContent(column.value.col_name, "number", True, False, closed_ended_multilingual_col=True, invariant=False)])
+
 auto_eval_column_dict.append(["is_domain_specific", ColumnContent, ColumnContent("Is Domain Specific", "bool", False)])
 auto_eval_column_dict.append(["use_chat_template", ColumnContent, ColumnContent("Uses Chat Template", "bool", False)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -75,6 +91,13 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 # auto_eval_column_dict.append(["display_result", ColumnContent, ColumnContent("Display Result", "bool", False, True)])
 auto_eval_column_dict.append(["date", ColumnContent, ColumnContent("Submission Date", "str", False)])
 
+# from dataclasses import make_dataclass, field
+
+# Example of fixing mutable defaults
+# auto_eval_column_dict = {
+#     "example_field": field(default_factory=dict),  # Replace mutable default
+#     "another_field": field(default_factory=list),  # Replace mutable default
+# }
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
@@ -94,8 +117,8 @@ class EvalQueueColumn:  # Queue column
     med_safety_status = ColumnContent("med_safety_status", "str", True)
     medical_summarization_status = ColumnContent("medical_summarization_status", "str", True)
     note_generation_status = ColumnContent("note_generation_status", "str", True)
-    if PRIVATE_REPO:
-        closed_ended_arabic_status = ColumnContent("closed_ended_arabic_status", "str", True)
+    # if PRIVATE_REPO:
+    #     closed_ended_arabic_status = ColumnContent("closed_ended_arabic_status", "str", True)
 
 ## All the model information that we might need
 @dataclass
@@ -221,8 +244,22 @@ MED_SAFETY_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c
 MEDICAL_SUMMARIZATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.medical_summarization_col or c.invariant)]
 ACI_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.aci_col or c.invariant)]
 SOAP_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.soap_col or c.invariant)]
+
+OpenEndedArabic_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_arabic_col or c.invariant)]
+OpenEndedFrench_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_french_col or c.invariant)]
+OpenEndedSpanish_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_spanish_col or c.invariant)]
+OpenEndedPortuguese_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_portuguese_col or c.invariant)]
+OpenEndedRomanian_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_romanian_col or c.invariant)]
+OpenEndedGreek_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.open_ended_greek_col or c.invariant)]
+
+
+
+ClosedEndedMultilingual_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.closed_ended_multilingual_col or c.invariant)]
+
+
+
 # if PRIVATE_REPO:
-CLOSED_ENDED_ARABIC_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.closed_ended_arabic_col or c.invariant)]
+#CLOSED_ENDED_ARABIC_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.closed_ended_arabic_col or c.invariant)]
 # CROSS_EXAMINATION_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and (c.cross_examination_col or c.invariant)]
 # DATASET_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.open_ended_col and not c.med_safety_col and not c.cross_examination_col]
 # OPEN_ENDED_COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.dataset_task_col and not c.med_safety_col and not c.cross_examination_col]
@@ -243,8 +280,26 @@ MED_SAFETY_BENCHMARK_COLS = [t.value.col_name for t in MedSafetyColumns]
 MEDICAL_SUMMARIZATION_BENCHMARK_COLS = [t.value.col_name for t in MedicalSummarizationColumns]
 ACI_BENCHMARK_COLS = [t.value.col_name for t in ACIColumns]
 SOAP_BENCHMARK_COLS = [t.value.col_name for t in SOAPColumns]
-# if PRIVATE_REPO:
-CLOSED_ENDED_ARABIC_BENCHMARK_COLS = [t.value.col_name for t in ClosedEndedArabicColumns]
+
+
+#changed this 
+OpenEndedArabic_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedArabicColumn]
+OpenEndedFrench_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedFrenchColumn]
+OpenEndedPortuguese_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedPortugueseColumn]
+OpenEndedSpanish_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedSpanishColumn]
+OpenEndedRomanian_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedRomanianColumn]
+OpenEndedGreek_BENCHMARK_COLS = [t.value.col_name for t in OpenEndedGreekColumn]
+
+
+ClosedEndedMultilingual_BENCHMARK_COLS = [t.value.col_name for t in ClosedEndedMultilingualColumns]
+
+
+
+
+
+
+# # if PRIVATE_REPO:
+# CLOSED_ENDED_ARABIC_BENCHMARK_COLS = [t.value.col_name for t in ClosedEndedArabicColumns]
 # CROSS_EXAMINATION_BENCHMARK_COLS = [t.value.col_name for t in CrossExaminationTasks]
 
 NUMERIC_INTERVALS = {

src/leaderboard/instr.txt

@@ -0,0 +1,16 @@
+in about
+from app, to read evals, to utils to about ( to define the tasks and the colums ( so for close-ended define the languages and for open-ended ( use the same code with 95%CI, Elo rating...)))
+define a class for open-ended-multilingual ( 6 times for all) the and close-ended mulitlingual globalmmlu
+6 columns for open-ended and one different for multili
+
+in utils: 
+
+i should define the columns for languages again ( here we dont care about the hidden parts but we need to define in the beginning  )
+
+in read_evals 
+
+definition of the results of the data frames, and the definition of the int
+
+for the front end: 
+
+in the app.py,i should add the gr.tabitem for open-ended, follow the healthbench and add the languages same logic as "ALL"

src/leaderboard/read_evals.py

@@ -9,7 +9,7 @@ import numpy as np
 
 from src.display.formatting import make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.display.utils import AutoEvalColumn, ModelType, ModelArch, Precision, HarnessTasks, WeightType, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedMultilingualColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn
 from src.submission.check_validity import is_model_on_hub
 from src.envs import PRIVATE_REPO
 
@@ -30,7 +30,13 @@ class EvalResult:
     medical_summarization_results: dict
     aci_results: dict
     soap_results: dict
-    closed_ended_arabic_results: dict
+    open_ended_arabic_results: dict
+    open_ended_french_results: dict
+    open_ended_spanish_results: dict
+    open_ended_portuguese_results: dict
+    open_ended_romanian_results: dict
+    open_ended_greek_results: dict
+    closed_ended_multilingual_results: dict
     is_domain_specific: bool
     use_chat_template: bool
     # clinical_type_results:dict
@@ -108,7 +114,7 @@ class EvalResult:
         open_ended_results = {}
         if "open-ended" in data["results"]:
             for task in OpenEndedColumns:
-                task = task.value
+                task = task.value 
                 # We average all scores of a given metric (not all metrics are present in all files)
                 accs = data["results"]["open-ended"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended"]["overall"] else None
                 open_ended_results[task.benchmark] = accs
@@ -167,20 +173,90 @@ class EvalResult:
                     continue
                 mean_acc = np.mean(accs)  # * 100.0
                 soap_results[task.benchmark] = mean_acc
-        closed_ended_arabic_results = {}
-        if PRIVATE_REPO and "closed-ended-arabic" in data["results"]:
-            for task in ClosedEndedArabicColumns:
+        open_ended_arabic_results = {}
+        if "open-ended-arabic" in data["results"]:
+            for task in OpenEndedArabicColumn:
                 task = task.value
                 # We average all scores of a given metric (not all metrics are present in all files)
-                try:
-                    accs = np.array([v.get(task.metric, None) for k, v in data["results"]["closed-ended-arabic"].items() if task.benchmark == k])
-                except:
-                    # breakpoint()
-                    accs = np.array([])
-                if accs.size == 0 or any([acc is None for acc in accs]):
-                    continue
-                mean_acc = np.mean(accs)  # * 100.0
-                closed_ended_arabic_results[task.benchmark] = mean_acc
+                accs = data["results"]["open-ended-arabic"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended-arabic"]["overall"] else None
+                open_ended_arabic_results[task.benchmark] = accs
+            if open_ended_arabic_results["ELO_intervals"] is not None and open_ended_arabic_results["Score_intervals"] is not None:
+                open_ended_arabic_results["ELO_intervals"] = "+" + str(open_ended_arabic_results["ELO_intervals"][1]) + "/-" + str(abs(float(open_ended_arabic_results["ELO_intervals"][0])))
+                open_ended_arabic_results["Score_intervals"] = "+" + str(open_ended_arabic_results["Score_intervals"][1]) + "/-" + str(abs(float(open_ended_arabic_results["Score_intervals"][0])))
+        open_ended_french_results = {}
+        if "open-ended-french" in data["results"]:
+            for task in OpenEndedFrenchColumn:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = data["results"]["open-ended-french"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended-french"]["overall"] else None
+                open_ended_french_results[task.benchmark] = accs
+            if open_ended_french_results["ELO_intervals"] is not None and open_ended_french_results["Score_intervals"] is not None:
+                open_ended_french_results["ELO_intervals"] = "+" + str(open_ended_french_results["ELO_intervals"][1]) + "/-" + str(abs(open_ended_french_results["ELO_intervals"][0]))
+                open_ended_french_results["Score_intervals"] = "+" + str(open_ended_french_results["Score_intervals"][1]) + "/-" + str(abs(open_ended_french_results["Score_intervals"][0]))
+        open_ended_spanish_results = {}
+        if "open-ended-spanish" in data["results"]:
+            for task in OpenEndedSpanishColumn:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = data["results"]["open-ended-spanish"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended-spanish"]["overall"] else None
+                open_ended_spanish_results[task.benchmark] = accs
+            if open_ended_spanish_results["ELO_intervals"] is not None and open_ended_spanish_results["Score_intervals"] is not None:
+                open_ended_spanish_results["ELO_intervals"] = "+" + str(open_ended_spanish_results["ELO_intervals"][1]) + "/-" + str(abs(open_ended_spanish_results["ELO_intervals"][0]))
+                open_ended_spanish_results["Score_intervals"] = "+" + str(open_ended_spanish_results["Score_intervals"][1]) + "/-" + str(abs(open_ended_spanish_results["Score_intervals"][0]))
+        open_ended_portuguese_results = {}
+        if "open-ended-portuguese" in data["results"]:
+            for task in OpenEndedPortugueseColumn:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = data["results"]["open-ended-portuguese"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended-portuguese"]["overall"] else None
+                open_ended_portuguese_results[task.benchmark] = accs
+            if open_ended_portuguese_results["ELO_intervals"] is not None and open_ended_portuguese_results["Score_intervals"] is not None:
+                open_ended_portuguese_results["ELO_intervals"] = "+" + str(open_ended_portuguese_results["ELO_intervals"][1]) + "/-" + str(abs(open_ended_portuguese_results["ELO_intervals"][0]))
+                open_ended_portuguese_results["Score_intervals"] = "+" + str(open_ended_portuguese_results["Score_intervals"][1]) + "/-" + str(abs(open_ended_portuguese_results["Score_intervals"][0]))
+        open_ended_romanian_results = {}
+        if "open-ended-romanian" in data["results"]:
+            for task in OpenEndedRomanianColumn:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = data["results"]["open-ended-romanian"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended-romanian"]["overall"] else None
+                open_ended_romanian_results[task.benchmark] = accs
+            if open_ended_romanian_results["ELO_intervals"] is not None and open_ended_romanian_results["Score_intervals"] is not None:
+                open_ended_romanian_results["ELO_intervals"] = "+" + str(open_ended_romanian_results["ELO_intervals"][1]) + "/-" + str(abs(open_ended_romanian_results["ELO_intervals"][0]))
+                open_ended_romanian_results["Score_intervals"] = "+" + str(open_ended_romanian_results["Score_intervals"][1]) + "/-" + str(abs(open_ended_romanian_results["Score_intervals"][0]))
+        open_ended_greek_results = {}
+        if "open-ended-greek" in data["results"]:
+            for task in OpenEndedGreekColumn:
+                task = task.value
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = data["results"]["open-ended-greek"]["overall"][task.benchmark] if task.benchmark in data["results"]["open-ended-greek"]["overall"] else None
+                open_ended_greek_results[task.benchmark] = accs
+            if open_ended_greek_results["ELO_intervals"] is not None and open_ended_greek_results["Score_intervals"] is not None:
+                open_ended_greek_results["ELO_intervals"] = "+" + str(open_ended_greek_results["ELO_intervals"][1]) + "/-" + str(abs(float(open_ended_greek_results["ELO_intervals"][0])))
+                open_ended_greek_results["Score_intervals"] = "+" + str(open_ended_greek_results["Score_intervals"][1]) + "/-" + str(abs(float(open_ended_greek_results["Score_intervals"][0])))
+        closed_ended_multilingual_results = {} 
+        if "closed-ended-multilingual" in data["results"]:
+            for task in  ClosedEndedMultilingualColumns:
+                task = task.value
+                accs = data["results"]["closed-ended-multilingual"][task.benchmark]["accuracy"] if task.benchmark in data["results"]["closed-ended-multilingual"] else None
+                closed_ended_multilingual_results[task.benchmark] = accs
+            
+        #         #add the 
+        # closed_ended_arabic_results = {}
+        # if PRIVATE_REPO and "closed-ended-arabic" in data["results"]:
+        #     for task in ClosedEndedArabicColumns:
+        #         task = task.value
+        #         # We average all scores of a given metric (not all metrics are present in all files)
+        #         try:
+        #             accs = np.array([v.get(task.metric, None) for k, v in data["results"]["closed-ended-arabic"].items() if task.benchmark == k])
+        #         except:
+        #             # breakpoint()
+        #             accs = np.array([])
+        #         if accs.size == 0 or any([acc is None for acc in accs]):
+        #             continue
+        #         mean_acc = np.mean(accs)  # * 100.0
+        #         closed_ended_arabic_results[task.benchmark] = mean_acc
+                
+                
         # if open_ended_results == {} or med_safety_results == {} or medical_summarization_results == {} or aci_results == {} or soap_results == {}:
         #     open_ended_results = {}
         #     med_safety_results = {}
@@ -212,7 +288,13 @@ class EvalResult:
             medical_summarization_results=medical_summarization_results,
             aci_results=aci_results,
             soap_results=soap_results,
-            closed_ended_arabic_results=closed_ended_arabic_results,
+            open_ended_arabic_results=open_ended_arabic_results,
+            open_ended_french_results=open_ended_french_results,
+            open_ended_spanish_results=open_ended_spanish_results,
+            open_ended_portuguese_results=open_ended_portuguese_results,
+            open_ended_romanian_results=open_ended_romanian_results,
+            open_ended_greek_results=open_ended_greek_results,
+            closed_ended_multilingual_results=closed_ended_multilingual_results,
             is_domain_specific=config.get("is_domain_specific", False),  # Assuming a default value
             use_chat_template=config.get("use_chat_template", False),  # Assuming a default value
             precision=precision,
@@ -315,12 +397,42 @@ class EvalResult:
                 for task in SOAPColumns:
                     data_dict[task.value.col_name] = self.soap_results[task.value.benchmark]
             return data_dict
-        if PRIVATE_REPO and subset == "closed_ended_arabic":
-            average = sum([v for v in self.closed_ended_arabic_results.values() if v is not None]) / len(ClosedEndedArabicColumns)
+        if subset == "open_ended_arabic":
+            if len(self.open_ended_arabic_results) > 0:
+                for task in OpenEndedArabicColumn:
+                    data_dict[task.value.col_name] = self.open_ended_arabic_results[task.value.benchmark]
+            return data_dict
+        if subset == "open_ended_french":
+            if len(self.open_ended_french_results) > 0:
+                for task in OpenEndedFrenchColumn:
+                    data_dict[task.value.col_name] = self.open_ended_french_results[task.value.benchmark]
+            return data_dict
+        if subset == "open_ended_spanish":
+            if len(self.open_ended_spanish_results) > 0:
+                for task in OpenEndedSpanishColumn:
+                    data_dict[task.value.col_name] = self.open_ended_spanish_results[task.value.benchmark]
+            return data_dict
+        if subset == "open_ended_portuguese":
+            if len(self.open_ended_portuguese_results) > 0:
+                for task in OpenEndedPortugueseColumn:
+                    data_dict[task.value.col_name] = self.open_ended_portuguese_results[task.value.benchmark]
+            return data_dict
+        if subset == "open_ended_romanian":
+            if len(self.open_ended_romanian_results) > 0:
+                for task in OpenEndedRomanianColumn:
+                    data_dict[task.value.col_name] = self.open_ended_romanian_results[task.value.benchmark]
+            return data_dict
+        if subset == "open_ended_greek":
+            if len(self.open_ended_greek_results) > 0:
+                for task in OpenEndedGreekColumn:
+                    data_dict[task.value.col_name] = self.open_ended_greek_results[task.value.benchmark]
+            return data_dict
+        if subset == "closed_ended_multilingual":
+            average = sum([v for v in self.closed_ended_multilingual_results.values() if v is not None]) / len(ClosedEndedMultilingualColumns)
             data_dict[AutoEvalColumn.average.name] = average
-            if len(self.closed_ended_arabic_results) > 0:
-                for task in ClosedEndedArabicColumns:
-                    data_dict[task.value.col_name] = self.closed_ended_arabic_results[task.value.benchmark]
+            if len(self.closed_ended_multilingual_results) > 0:
+                for task in ClosedEndedMultilingualColumns:
+                    data_dict[task.value.col_name] = self.closed_ended_multilingual_results[task.value.benchmark]
             return data_dict
 
 def get_request_file_for_model(requests_path, model_name, precision):

src/populate.py

@@ -5,7 +5,7 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 # changes to be made here
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, ClosedEndedArabicColumns
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, OpenEndedColumns, MedSafetyColumns, MedicalSummarizationColumns, ACIColumns, SOAPColumns, OpenEndedArabicColumn, OpenEndedFrenchColumn, OpenEndedSpanishColumn, OpenEndedPortugueseColumn, OpenEndedRomanianColumn, OpenEndedGreekColumn, ClosedEndedMultilingualColumns
 from src.leaderboard.read_evals import get_raw_eval_results
 from src.envs import PRIVATE_REPO
 
@@ -16,14 +16,15 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     # print(raw_data)
     # raise Exception("stop")
     all_data_json = [v.to_dict(subset=subset) for v in raw_data]
-
+    # if subset.startswith("open_ended"):
+        # breakpoint()
     df = pd.DataFrame.from_records(all_data_json)
     # changes to be made here
     if subset == "datasets":
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     elif subset == "med_safety":
         df = df.sort_values(by=["Harmfulness Score"], ascending=True)
-    elif subset == "open_ended":
+    elif subset.startswith("open_ended"):
         df = df.sort_values(by=["ELO"], ascending=False)
     elif subset == "medical_summarization":
         df = df.sort_values(by=[AutoEvalColumn.overall.name], ascending=False)
@@ -31,7 +32,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
         df = df.sort_values(by=[AutoEvalColumn.overall.name], ascending=False)
     elif subset == "soap":
         df = df.sort_values(by=[AutoEvalColumn.overall.name], ascending=False)
-    elif subset == "closed_ended_arabic":
+    elif subset == "closed_ended_multilingual":
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     cols = list(set(df.columns).intersection(set(cols)))
     df = df[cols].round(decimals=2)