chore: Reorder arguments in run_model_task function

.gitignore

@@ -1,3 +1,5 @@
+results/
+
 # Created by https://www.toptal.com/developers/gitignore/api/python,visualstudiocode,macos,windows
 # Edit at https://www.toptal.com/developers/gitignore?templates=python,visualstudiocode,macos,windows
 

calculate_memory_usage.py

@@ -1,7 +1,5 @@
-import json
 import os
 import glob
-import numpy as np
 import argparse
 from code_efficiency_calculator import run_model_task
 
@@ -52,8 +50,8 @@ def report_max_memory_usage(dat_file_path):
             max_memory_usage = max(max_memory_usage, mem_in_mb)
         return max_memory_usage
 
-def report_results(task,model):
-    run_model_task(task,model)
+def report_results(task, model, file):
+    run_model_task(task, model, file)
     dat_directory = f"./results/{task}_{model}"
     canonical_solution_directory = f"./results/{task}_canonical_solution"
     canonical_solution_memory_usage = {}
@@ -209,12 +207,17 @@ def report_results(task,model):
         total_500_nmu = total_500_nmu/len(normalized_execution_time_list)*100
         total_500_tmu = total_500_tmu/len(normalized_execution_time_list)*100
 
-        print(f"{model}&{total_execution_time:.2f}&{normalized_execution_time:.2f}&{max_net:.2f}&{total_500_net:.1f}&{total_max_memory_usage:.2f}&{normalized_max_memory_usage:.2f}&{max_nmu:.2f}&{total_500_nmu:.1f}&{total_memory_usage:.2f}&{normalized_memory_usage:.2f}&{max_tmu:.2f}&{total_500_tmu:.1f}&{pass1:.1f}\\\\")
+        return f"{model}&{total_execution_time:.2f}&{normalized_execution_time:.2f}&{max_net:.2f}&{total_500_net:.1f}&{total_max_memory_usage:.2f}&{normalized_max_memory_usage:.2f}&{max_nmu:.2f}&{total_500_nmu:.1f}&{total_memory_usage:.2f}&{normalized_memory_usage:.2f}&{max_tmu:.2f}&{total_500_tmu:.1f}&{pass1:.1f}\\\\"
+
 
 if __name__ == "__main__":
     parse = argparse.ArgumentParser()
     parse.add_argument("--task", type=str, default="EffiBench")
     parse.add_argument("--model", type=str, default="gpt-4")
-
+    parse.add_argument("--file", type=str, default="")
     args = parse.parse_args()
-    report_results(args.task,args.model)
+
+    if not args.file:
+        args.file = f"./{args.task}_{args.model}.json"
+    
+    report_results(args.task,args.model, args.file)

code_efficiency_calculator.py

@@ -11,22 +11,12 @@ import os
 import re
 import shutil
 import contextlib
-import random
-import json
-from typing import Optional, Callable, Dict
-from concurrent.futures import ThreadPoolExecutor, as_completed
-import inspect
-import numpy as np
-import sys
+from concurrent.futures import ThreadPoolExecutor
 import concurrent.futures
-import time
 from tqdm import tqdm
 import contextlib
-import faulthandler
 import io
 import os
-import multiprocessing
-import platform
 import signal
 from tqdm import tqdm
 
@@ -618,13 +608,14 @@ def fetch_completion(dataset,model):
     return dataset
 
 
-def run_model_task(task,model):
+def run_model_task(task, model, file):
 
     if "/" in model:
         model = model.split("/")[1]
     dat_path = f"./results/{task}_{model}"
     canonical_solution_path = f"./results/{task}_canonical_solution"
-    with open(f"./{task}_{model}.json", "r") as f:
+    
+    with open(file, "r") as f:
         dataset = json.load(f)
 
     if os.path.exists(dat_path):
@@ -639,7 +630,7 @@ def run_model_task(task,model):
 
     fetch_completion(dataset,dat_path)
 
-    with open(f"./{task}_{model}.json", "r") as f:
+    with open(file, "r") as f:
         dataset = json.load(f)
     for i in range(len(dataset)):
         dataset[i]["dataset"] = f"{task}"
@@ -647,9 +638,14 @@ def run_model_task(task,model):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model", type=str, default="gpt-4")
-    parser.add_argument("--task", type=str, default="EffiBench")
-    args = parser.parse_args()
+    parse = argparse.ArgumentParser()
+    parse.add_argument("--task", type=str, default="EffiBench")
+    parse.add_argument("--model", type=str, default="gpt-4")
+    parse.add_argument("--file", type=str, default="")
+    args = parse.parse_args()
+
+    if not args.file:
+        args.file = f"./{args.task}_{args.model}.json"
+
+    run_model_task(args.task, args.model, args.file)
 
-    run_model_task(args.model,args.task)

leaderboard.py

@@ -63,8 +63,7 @@ def process_uploaded_file(file):
     except Exception as e:
         return f"Error parsing the task and model name from the file name: {str(e)}! Should be in the format of <task>_<model>.json"
     
-    
-
+    return report_results(task, model, file)
 
 def build_leaderboard_tab(leaderboard_table_file):
     gr.Markdown(make_default_md_1(), elem_id="leaderboard_markdown")

requirements.txt

@@ -1,3 +1,6 @@
+tqdm
+numpy
+gradio
 plotly
 line_profiler
 memory_profiler