import json import os import pandas as pd def get_new_leaderboard_df(results_path: str) -> pd.DataFrame: model_result_filepaths = [] for root, _, files in os.walk(results_path): if len(files) == 0 or any([not f.endswith(".json") for f in files]): continue for file in files: model_result_filepaths.append(os.path.join(root, file)) eval_results = { 'model': [], 'buzz_accuracy': [], 'win_rate_human': [], 'win_rate_model': [] } for model_result_filepath in model_result_filepaths: with open(model_result_filepath, "r") as fin: model_result = json.load(fin) model_id = model_result["model_id"] buzz_accuracy = model_result["buzz_accuracy"] win_rate_human = model_result["win_rate_human"] win_rate_model = model_result["win_rate_model"] eval_results['model'].append(model_id) eval_results['buzz_accuracy'].append(buzz_accuracy) eval_results['win_rate_human'].append(win_rate_human) eval_results['win_rate_model'].append(win_rate_model) return pd.DataFrame(eval_results)