leaderboard pipiline

app.py

@@ -33,20 +33,20 @@ from src.submission.submit import add_new_eval, upload_file
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=TOKEN)
 
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
-    )
-except Exception:
-    restart_space()
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print(EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+#     )
+# except Exception:
+#     restart_space()
 
 
 raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
@@ -56,86 +56,7 @@ value=[ c.name for c in fields(AutoEvalColumn)
 
 leaderboard_df = original_df.copy()
 
-# (
-#     finished_eval_queue_df,
-#     running_eval_queue_df,
-#     pending_eval_queue_df,
-# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-
-# Searching and filtering
-# def update_table(
-#     hidden_df: pd.DataFrame,
-#     columns: list,
-#     type_query: list,
-#     precision_query: str,
-#     size_query: list,
-#     show_deleted: bool,
-#     query: str,
-# ):
-#     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
-#     filtered_df = filter_queries(query, filtered_df)
-#     df = select_columns(filtered_df, columns)
-#     return df
-
-
-# def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-#     return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-
-
-# def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
-#     always_here_cols = [
-#         AutoEvalColumn.model_type_symbol.name,
-#         AutoEvalColumn.model.name,
-#     ]
-#     # We use COLS to maintain sorting
-#     filtered_df = df[
-#         always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
-#     ]
-#     return filtered_df
-
-
-# def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
-#     final_df = []
-#     if query != "":
-#         queries = [q.strip() for q in query.split(";")]
-#         for _q in queries:
-#             _q = _q.strip()
-#             if _q != "":
-#                 temp_filtered_df = search_table(filtered_df, _q)
-#                 if len(temp_filtered_df) > 0:
-#                     final_df.append(temp_filtered_df)
-#         if len(final_df) > 0:
-#             filtered_df = pd.concat(final_df)
-#             filtered_df = filtered_df.drop_duplicates(
-#                 subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
-#             )
-
-#     return filtered_df
-
-
-# def filter_models(
-#     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-# ) -> pd.DataFrame:
-#     # Show all models
-#     if show_deleted:
-#         filtered_df = df
-#     else:  # Show only still on the hub models
-#         filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
-#     type_emoji = [t[0] for t in type_query]
-#     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
-#     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
-#     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-#     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-#     mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
-#     filtered_df = filtered_df.loc[mask]
-
-#     return filtered_df
-
-
-# print([c.name for c in fields(AutoEvalColumn) if c.never_hidden])
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -143,58 +64,7 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            # with gr.Row():
-            #     with gr.Column():
-            #         with gr.Row():
-            #             search_bar = gr.Textbox(
-            #                 placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-            #                 show_label=False,
-            #                 elem_id="search-bar",
-            #             )
-            #         with gr.Row():
-            #             shown_columns = gr.CheckboxGroup(
-            #                 choices=[
-            #                     c.name
-            #                     for c in fields(AutoEvalColumn)
-            #                     if not c.hidden and not c.never_hidden and not c.dummy
-            #                 ],
-                            # value=[
-                            #     c.name
-                            #     for c in fields(AutoEvalColumn)
-                            #     if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            # ],
-            #                 label="Select columns to show",
-            #                 elem_id="column-select",
-            #                 interactive=True,
-            #             )
-            #         with gr.Row():
-            #             deleted_models_visibility = gr.Checkbox(
-            #                 value=False, label="Show gated/private/deleted models", interactive=True
-            #             )
-            #     with gr.Column(min_width=320):
-            #         #with gr.Box(elem_id="box-filter"):
-            #         filter_columns_type = gr.CheckboxGroup(
-            #             label="Model types",
-            #             choices=[t.to_str() for t in ModelType],
-            #             value=[t.to_str() for t in ModelType],
-            #             interactive=True,
-            #             elem_id="filter-columns-type",
-            #         )
-            #         filter_columns_precision = gr.CheckboxGroup(
-            #             label="Precision",
-            #             choices=[i.value.name for i in Precision],
-            #             value=[i.value.name for i in Precision],
-            #             interactive=True,
-            #             elem_id="filter-columns-precision",
-            #         )
-            #         filter_columns_size = gr.CheckboxGroup(
-            #             label="Model sizes (in billions of parameters)",
-            #             choices=list(NUMERIC_INTERVALS.keys()),
-            #             value=list(NUMERIC_INTERVALS.keys()),
-            #             interactive=True,
-            #             elem_id="filter-columns-size",
-            #         )
-
+           
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
                     [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + value
@@ -215,34 +85,7 @@ with demo:
                 datatype=TYPES,
                 visible=False,
             )
-            # search_bar.submit(
-            #     update_table,
-            #     [
-            #         hidden_leaderboard_table_for_search,
-            #         shown_columns,
-            #         filter_columns_type,
-            #         filter_columns_precision,
-            #         filter_columns_size,
-            #         deleted_models_visibility,
-            #         search_bar,
-            #     ],
-            #     leaderboard_table,
-            # )
-            # for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
-            #     selector.change(
-            #         update_table,
-            #         [
-            #             hidden_leaderboard_table_for_search,
-            #             shown_columns,
-            #             filter_columns_type,
-            #             filter_columns_precision,
-            #             filter_columns_size,
-            #             deleted_models_visibility,
-            #             search_bar,
-            #         ],
-            #         leaderboard_table,
-            #         queue=True,
-            #     )
+            
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
@@ -252,89 +95,26 @@ with demo:
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
-                # with gr.Column():
-                #     with gr.Accordion(
-                #         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             finished_eval_table = gr.components.Dataframe(
-                #                 value=finished_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
-                #     with gr.Accordion(
-                #         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             running_eval_table = gr.components.Dataframe(
-                #                 value=running_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
-
-                #     with gr.Accordion(
-                #         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                #         open=False,
-                #     ):
-                #         with gr.Row():
-                #             pending_eval_table = gr.components.Dataframe(
-                #                 value=pending_eval_queue_df,
-                #                 headers=EVAL_COLS,
-                #                 datatype=EVAL_TYPES,
-                #                 row_count=5,
-                #             )
+               
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your files here!", elem_classes="markdown-text")
 
+            def update_leaderboard(file_obj):
+                upload_file(file_obj)
+                raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+                value=[ c.name for c in fields(AutoEvalColumn) 
+                    if c.displayed_by_default and not c.hidden and not c.never_hidden]
+                        
+                leaderboard_df = original_df.copy()
+                leaderboard_table = leaderboard_df[
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden] + value
+                    + [AutoEvalColumn.dummy.name]
+                ]
+                return leaderboard_table
+                
             with gr.Row():
-                upload = gr.Interface(fn=upload_file,inputs="file" ,outputs=None)
-                # with gr.Column():
-                #     model_name_textbox = gr.Textbox(label="Model name")
-                #     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                #     model_type = gr.Dropdown(
-                #         choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                #         label="Model type",
-                #         multiselect=False,
-                #         value=None,
-                #         interactive=True,
-                #     )
-
-                # with gr.Column():
-                #     precision = gr.Dropdown(
-                #         choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                #         label="Precision",
-                #         multiselect=False,
-                #         value="float16",
-                #         interactive=True,
-                #     )
-                #     weight_type = gr.Dropdown(
-                #         choices=[i.value.name for i in WeightType],
-                #         label="Weights type",
-                #         multiselect=False,
-                #         value="Original",
-                #         interactive=True,
-                #     )
-                #     base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            # submit_button = gr.Button("Submit Eval")
-            # submission_result = gr.Markdown()
-            # submit_button.click(
-            #     add_new_eval,
-            #     [
-            #         model_name_textbox,
-            #         base_model_name_textbox,
-            #         revision_name_textbox,
-            #         precision,
-            #         weight_type,
-            #         model_type,
-            #     ],
-            #     submission_result,
-            # )
-
+                upload = gr.Interface(fn=update_leaderboard,inputs="file" ,outputs=leaderboard_table)
+               
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -347,6 +127,6 @@ with demo:
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=30)
-scheduler.start()
+# scheduler.start()
 
 demo.queue(default_concurrency_limit=40).launch()

src/display/about.py

@@ -11,8 +11,11 @@ class Task:
 # Init: to update with your specific keys
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("task_name1", "metric_name", "First task")
-    task1 = Task("task_name2", "metric_name", "Second task")
+    task0 = Task("fetaqa", "bleu", "fetaqa")
+    task1 = Task("hitab_ng", "bleu", "hitab_ng")
+    task2 = Task("qtsumm", "bleu", "qtsumm")
+    task3 = Task("scigen", "bleu", "scigen")
+    task4 = Task("totto", "bleu", "totto")
 
 
 # Your leaderboard name
@@ -25,43 +28,54 @@ Intro text
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## TEST 
+##  Introduction
+### OPENT2T - first open-source toolkit for table-to-text generation 
+- designed to reproduce existing table pre-training models for
+    - expedite the development of new models.
+    - performance comparison 
+- implemented and compared
+    - 19 types of large language models under zero and few-shot settings
+    - 7 fine-tuned models
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+Currently we released evaluation results on "fetaqa", "hitab_ng" , "qtsumm", "scigen", "totto" dataset
+
+##  Notice
+Currently, we only display local evaluation results on the leaderboard. We plan to add automatic evaluation scripts in the future. At that point, you will be able to upload your model output for evaluation.
 
 """
 
 EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+## Your submit file format
+Currently, we only accept the final results files. Your submission file should be a JSON file in the following format:
+```json
+{
+    "config": {
+        "model_dtype": <model_dtype>, 
+        "model_name":  <model_name>,
+    },
+    "results": {
+        "task_name": {
+            "bleu": <score>
+        },
+        
+        ...
+    }
+}
 ```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
 
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+## Please make sure
+#### 1)Your uploaded file is in the requested format
+Failure to do so may lead to unprecedented errors.
+#### 2) Your results cover all tasks.
+If some tasks do not have results, please assign a score of zero to those categories.
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
+CITATION_BUTTON_TEXT =r"""
+@article{zhang2024opent2t,
+  title={OPENT2T: An Open-Source Toolkit for Table-to-Text Generation},
+  author={Zhang, Haowei and Si, Shengyun and Zhao, Yilun and Wang, Pengcheng and Nan, Linyong and Tang, Xiangru and Radev, Dragomir and Cohan, Arman},
+  journal={ACL},
+  year={2024},
+}
 """

src/display/utils.py

@@ -69,7 +69,7 @@ class ModelType(Enum):
     FT = ModelDetails(name="fine-tuned", symbol="🔶")
     IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
     RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
+    Unknown = ModelDetails(name="", symbol="★")
 
     def to_str(self, separator=" "):
         return f"{self.value.symbol}{separator}{self.value.name}"

src/envs.py

@@ -8,8 +8,8 @@ TOKEN = os.environ.get("TOKEN", None)
 OWNER = "huohua0314"
 TEST  = "demo-leaderboard"
 REPO_ID = f"{OWNER}/OPENT2T"
-QUEUE_REPO = f"{TEST}/requests"
-RESULTS_REPO = f"{TEST}/results"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/opent2t_results"
 
 CACHE_PATH=os.getenv("HF_HOME", ".")
 

src/leaderboard/read_evals.py

@@ -107,14 +107,9 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        print(Tasks)
-        for item in Tasks:
-            print(item)
+
         
-        print(AutoEvalColumn.model_type_symbol.name)
-        print(AutoEvalColumn.model.name)
-        print(AutoEvalColumn.dummy.name)
-        print(AutoEvalColumn.average.name)
+  
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
@@ -133,11 +128,9 @@ class EvalResult:
             # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
         
-        print("zzzz")
-        print(self.results)
+      
         for task in Tasks:
             data_dict[task.value.col_name] = self.results[task.value.benchmark]
-        print("tttttzzzz")
 
         return data_dict
 
@@ -169,7 +162,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     model_result_filepaths = []
    
     for root, _, files in os.walk(results_path):
-        print(files)
+ 
         # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
@@ -197,16 +190,11 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             eval_results[eval_name] = eval_result
 
     results = []
-    # print(eval_results)
     for v in eval_results.values():
-        print(f"v : {v}")
         try:
             v.to_dict() # we test if the dict version is complete
-            print("aaa")
             results.append(v)
         except KeyError as e:  # not all eval values present
-            print(f"error : {e}")
-            print("nono")
             continue
 
     return results

src/populate.py

@@ -14,8 +14,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    print(df)
-    print(AutoEvalColumn.average.name)
+
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
 
     df = df[cols].round(decimals=2)