change to MASE and geometric mean to aggregate results

app.py

@@ -119,15 +119,15 @@ def init_leaderboard(ori_dataframe, model_info_df, sort_val: str|None = None):
     merged_df = get_merged_df(ori_dataframe, model_info_df)
     new_cols = ['T'] + [col for col in merged_df.columns if col != 'T']
     merged_df = merged_df[new_cols]
-    print('Merged df: ', merged_df)
     if sort_val:
         if sort_val in merged_df.columns:
             merged_df = merged_df.sort_values(by=[sort_val])
         else:
             print(f'Warning: cannot sort by {sort_val}')
+    print('Merged df: ', merged_df)
     # get the data type
     datatype_list = [col2type_dict[col] if col in col2type_dict else 'number' for col in merged_df.columns]
-    print('datatype_list: ', datatype_list)
+    # print('datatype_list: ', datatype_list)
     # print('merged_df.column: ', merged_df.columns)
     # ipdb.set_trace()
     return Leaderboard(
@@ -164,7 +164,7 @@ def init_leaderboard(ori_dataframe, model_info_df, sort_val: str|None = None):
             ColumnFilter(ModelInfoColumn.model_type.name, type="checkboxgroup", label="Model types"),
         ],
         # bool_checkboxgroup_label="",
-        column_widths=[40, 150] + [150 for _ in range(len(merged_df.columns)-2)],
+        column_widths=[40, 150] + [180 for _ in range(len(merged_df.columns)-2)],
         interactive=False,
     )
 
@@ -176,7 +176,7 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem('🏅 Overall', elem_id="llm-benchmark-tab-table", id=5):
-            leaderboard = init_leaderboard(overall_df, model_info_df, sort_val='MAPE')
+            leaderboard = init_leaderboard(overall_df, model_info_df, sort_val='Rank')
             print(f'FINAL Overall LEADERBOARD {overall_df}')
         with gr.TabItem("🏅 By Domain", elem_id="llm-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(domain_df, model_info_df)

requirements.txt

@@ -14,4 +14,5 @@ tqdm
 transformers
 tokenizers>=0.15.0
 sentencepiece
-ipdb
+ipdb
+scipy

src/utils.py

@@ -2,6 +2,7 @@ import ipdb
 import pandas as pd
 import os
 import re
+from scipy import stats
 
 # Define the formatting function
 def format_number(num):
@@ -45,7 +46,7 @@ def pivot_df(file_name, tab_name):
 
 def rename_metrics(df):
     df = df.rename(columns={
-        'eval_metrics/MAPE[0.5]': 'MAPE',
+        'eval_metrics/MASE[0.5]': 'MASE',
         'eval_metrics/mean_weighted_sum_quantile_loss': 'CRPS',
         'rank': 'Rank'
     })
@@ -89,7 +90,7 @@ def pivot_existed_df(df, tab_name):
     print('columns', df.columns)
     df_melted = pd.melt(df, id_vars=[tab_name, 'model'], var_name='metric', value_name='value')
     df_melted['metric'] = df_melted['metric'].replace({
-        'eval_metrics/MAPE[0.5]': 'MAPE',
+        'eval_metrics/MASE[0.5]': 'MASE',
         'eval_metrics/mean_weighted_sum_quantile_loss': 'CRPS',
         'rank': 'Rank',
     })
@@ -168,9 +169,12 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie
     df['univariate'] = df['num_variates'] == 1
 
     # group by domain
-    METRIC_CHOICES = ["eval_metrics/MAPE[0.5]", "eval_metrics/mean_weighted_sum_quantile_loss", "rank"]
+    METRIC_CHOICES = ["eval_metrics/MASE[0.5]", "eval_metrics/mean_weighted_sum_quantile_loss"]
+    # ipdb.set_trace()
+    grouped_results_overall = df.groupby(['model'])[METRIC_CHOICES].agg(stats.gmean)
+    grouped_results_overall_rank = df.groupby(['model'])[['rank']].mean()
+    grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_rank], axis=1)
 
-    grouped_results_overall = df.groupby(['model'])[METRIC_CHOICES].mean()
     # grouped_results_overall = grouped_results_overall.rename(columns={'model':'Model'})
     # grouped_results.to_csv(f'artefacts/grouped_results_by_model.csv')
     grouped_dfs = {}
@@ -236,8 +240,10 @@ def standardize_df(df):
     return df
 
 def group_by(df, col_name):
-    METRIC_CHOICES = ["eval_metrics/MAPE[0.5]", "eval_metrics/mean_weighted_sum_quantile_loss", "rank"]
-    grouped_results = df.groupby([col_name, 'model'])[METRIC_CHOICES].mean()
+    METRIC_CHOICES = ["eval_metrics/MASE[0.5]", "eval_metrics/mean_weighted_sum_quantile_loss"]
+    grouped_results = df.groupby([col_name, 'model'])[METRIC_CHOICES].agg(stats.gmean)
+    grouped_results_rank = df.groupby([col_name, 'model'])[['rank']].mean()
+    grouped_results = pd.concat([grouped_results, grouped_results_rank], axis=1)
     # Display the results
     # Write the results to a csv file
     # grouped_results.to_csv(f'grouped_results_by_{col_name}.csv')