import gradio as gr
import subprocess
import sys
import os
import threading
import time
import uuid
import glob
import shutil
from pathlib import Path
default_command = "bigcodebench.evaluate"
# is_running = False
def generate_command(
jsonl_file, split, subset, parallel,
min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
check_gt_only, no_gt
):
command = [default_command]
if jsonl_file is not None:
# Copy the uploaded file to the current directory
local_filename = os.path.basename(jsonl_file.name)
shutil.copy(jsonl_file.name, local_filename)
command.extend(["--samples", local_filename])
command.extend(["--split", split, "--subset", subset])
if parallel is not None and parallel != 0:
command.extend(["--parallel", str(int(parallel))])
command.extend([
"--min-time-limit", str(min_time_limit),
"--max-as-limit", str(int(max_as_limit)),
"--max-data-limit", str(int(max_data_limit)),
"--max-stack-limit", str(int(max_stack_limit))
])
if check_gt_only:
command.append("--check-gt-only")
if no_gt:
command.append("--no-gt")
return " ".join(command)
def cleanup_previous_files(jsonl_file):
if jsonl_file is not None:
file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
else:
file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
for file in glob.glob("*"):
try:
if file not in file_list and not file.startswith("bigcode"):
os.remove(file)
except Exception as e:
print(f"Error during cleanup of {file}: {e}")
def find_result_file():
json_files = glob.glob("*.json")
if json_files:
return max(json_files, key=os.path.getmtime)
return None
def run_bigcodebench(command):
# global is_running
# if is_running:
# yield "A command is already running. Please wait for it to finish.\n"
# return
# is_running = True
# try:
yield f"Executing command: {command}\n"
process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
def kill_process():
if process.poll() is None: # If the process is still running
process.terminate()
# is_running = False
yield "Process terminated after 12 minutes timeout.\n"
# Start a timer to kill the process after 12 minutes
timer = threading.Timer(720, kill_process)
timer.start()
for line in process.stdout:
yield line
# process.wait()
timer.cancel()
if process.returncode != 0:
yield f"Error: Command exited with status {process.returncode}\n"
yield "Evaluation completed.\n"
result_file = find_result_file()
if result_file:
yield f"Result file found: {result_file}\n"
else:
yield "No result file found.\n"
# finally:
# is_running = False
def stream_logs(command, jsonl_file=None):
# global is_running
# if is_running:
# yield "A command is already running. Please wait for it to finish.\n"
# return
cleanup_previous_files(jsonl_file)
yield "Cleaned up previous files.\n"
log_content = []
for log_line in run_bigcodebench(command):
log_content.append(log_line)
yield "".join(log_content)
# with gr.Blocks() as demo:
# gr.Markdown("# BigCodeBench Evaluator")
# with gr.Row():
# jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
# split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
# subset = gr.Dropdown(choices=["hard", "full"], label="Subset", value="hard")
# with gr.Row():
# parallel = gr.Number(label="Parallel (optional)", precision=0)
# min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
# max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
# with gr.Row():
# max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
# max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
# check_gt_only = gr.Checkbox(label="Check GT Only")
# no_gt = gr.Checkbox(label="No GT")
# command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
# with gr.Row():
# submit_btn = gr.Button("Run Evaluation")
# download_btn = gr.DownloadButton(label="Download Result")
# log_output = gr.Textbox(label="Execution Logs", lines=20)
# input_components = [
# jsonl_file, split, subset, parallel,
# min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
# check_gt_only, no_gt
# ]
# for component in input_components:
# component.change(generate_command, inputs=input_components, outputs=command_output)
# def start_evaluation(command, jsonl_file, subset, split):
# extra = subset + "_" if subset != "full" else ""
# if jsonl_file is not None:
# result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
# else:
# result_path = None
# for log in stream_logs(command, jsonl_file):
# if jsonl_file is not None:
# yield log, gr.update(value=result_path, label=result_path), gr.update()
# else:
# yield log, gr.update(), gr.update()
# result_file = find_result_file()
# if result_file:
# return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
# # gr.Button(visible=False)#,
# # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
# else:
# return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
# # gr.Button("Run Evaluation", visible=True),
# # gr.DownloadButton(visible=False))
# submit_btn.click(start_evaluation,
# inputs=[command_output, jsonl_file, subset, split],
# outputs=[log_output, download_btn])
# REPO_ID = "bigcode/bigcodebench-evaluator"
# HF_TOKEN = os.environ.get("HF_TOKEN", None)
# API = HfApi(token=HF_TOKEN)
# def restart_space():
# API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
# demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
# scheduler = BackgroundScheduler()
# scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
# scheduler.start()