src/populate.py

import json
import os

import pandas as pd

def get_new_leaderboard_df(results_path: str) -> pd.DataFrame:
    model_result_filepaths = []
    for root, _, files in os.walk(results_path):
        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
            continue
        for file in files:
            model_result_filepaths.append(os.path.join(root, file))

    eval_results = {
        'model': [],
        'buzz_accuracy': [],
        'win_rate_human': [],
        'win_rate_model': []
    }
    for model_result_filepath in model_result_filepaths:
        with open(model_result_filepath, "r") as fin:
            model_result = json.load(fin)
            model_id = model_result["model_id"]
            buzz_accuracy = model_result["buzz_accuracy"]
            win_rate_human = model_result["win_rate_human"]
            win_rate_model = model_result["win_rate_model"]
            eval_results['model'].append(model_id)
            eval_results['buzz_accuracy'].append(buzz_accuracy)
            eval_results['win_rate_human'].append(win_rate_human)
            eval_results['win_rate_model'].append(win_rate_model)
    return pd.DataFrame(eval_results)