Asta Leaderboard First Draft (#3)

app.py

@@ -1,644 +1,104 @@
-"""app.py: Gradio app for the AstaBench leaderboard.
-
-Modeled after the GAIA huggingface leaderboard app.
-
-"""
-
-import json
+# app.py
+import gradio as gr
 import os
-import shutil
-import tarfile
-import tempfile
-from datetime import datetime, timedelta, timezone
-from email.utils import parseaddr
-from pathlib import Path
-from zoneinfo import ZoneInfo
 
-import gradio as gr
-import numpy as np
-import pandas as pd
-import requests
-from agenteval import (
-    compute_summary_statistics,
-    process_eval_logs,
-    upload_folder_to_hf,
-    upload_summary_to_hf,
-)
-from agenteval.models import EvalResult
-from agenteval.upload import sanitize_path_component
 from apscheduler.schedulers.background import BackgroundScheduler
-from datasets import Dataset, DatasetDict, VerificationMode, load_dataset
-from datasets.data_files import EmptyDatasetError
 from huggingface_hub import HfApi
+import literature_understanding, main_page, c_and_e, data_analysis, e2e
 
-from content import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    INTRODUCTION_TEXT,
-    SUBMISSION_TEXT,
-    TITLE,
-    format_error,
-    format_log,
-    format_warning,
-    hf_uri_to_web_url,
-    hyperlink,
-)
+from content import TITLE, css
 
-# Should be False on spaces and True outside
+# --- Constants and Configuration  ---
 LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
-
-
-CONFIG_NAME = "1.0.0-dev1"
-
 IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
-
 OWNER = "allenai"
 PROJECT_NAME = "asta-bench" + ("-internal" if IS_INTERNAL else "")
-SUBMISSION_DATASET = f"{OWNER}/{PROJECT_NAME}-submissions"  # all raw and scored submissions (val and test)
-SUBMISSION_DATASET_PUBLIC = f"{OWNER}/{PROJECT_NAME}-submissions-public"  # copy scored val submissions (public for transparency - not used for rendering leaderboard)
-CONTACT_DATASET = f"{OWNER}/{PROJECT_NAME}-contact-info"
-RESULTS_DATASET = f"{OWNER}/{PROJECT_NAME}-results"  # just the summary score statistics (val and test), to be displayed on the leaderboard
 LEADERBOARD_PATH = f"{OWNER}/{PROJECT_NAME}-leaderboard"
-
-if LOCAL_DEBUG:
-    DATA_DIR = os.path.join(os.path.dirname(__file__), "data", CONFIG_NAME)
-else:
-    DATA_DIR = "/home/user/data/" + CONFIG_NAME
-EXTRACTED_DATA_DIR = os.path.join(DATA_DIR, "extracted")
-
 api = HfApi()
-
-# max upload size of 100MB
-MAX_UPLOAD_BYTES = 100 * 1024**2
-
-AGENTEVAL_MANIFEST_NAME = "agenteval.json"
-
-
-os.makedirs(EXTRACTED_DATA_DIR, exist_ok=True)
-
-
-def try_load_dataset(*args, **kwargs) -> DatasetDict:
-    try:
-        return load_dataset(*args, **kwargs)
-    except EmptyDatasetError:
-        return DatasetDict()
-    except ValueError:
-        return DatasetDict()
-
-
-def pretty_column_name(col: str) -> str:
-    """Map any raw column name to its display name."""
-    # text columns
-    if col == "submit_time":
-        return "Submission date"
-    elif col == "agent_name":
-        return "Agent"
-    elif col == "agent_description":
-        return "Agent description"
-    elif col == "username":
-        return "User/organization"
-    elif col == "logs_url":
-        return "Logs"
-    # cost → $
-    if col.endswith("/cost"):
-        return "$"
-    # stderr → CI
-    elif col.endswith("/cost_stderr") or col.endswith("/score_stderr"):
-        return "CI"
-    # overall score
-    elif col == "overall/score":
-        return "Overall"
-    # any other score → its tag/task name
-    elif col.endswith("/score"):
-        return col.split("/")[1]
-    # fallback to unchanged
-    return col
-
-
-def get_dataframe_from_results(eval_results: DatasetDict, split: str):
-    local_df = eval_results.get(split)
-    # return default if split is missing or contains no records
-    if local_df is None or len(local_df) == 0:
-        default_raw_cols = [
-            "agent_name",
-            "agent_description",
-            "username",
-            "submit_time",
-        ]
-        pretty_cols = [pretty_column_name(c) for c in default_raw_cols]
-        return pd.DataFrame({col: ["No data"] for col in pretty_cols})
-
-    # Use the first suite_config for all rows
-    # because the suite_config should not change given a single CONFIG_NAME
-    first_suite_config = None
-    if len(local_df) > 0:
-        first_suite_config = EvalResult.model_validate(local_df[0]).suite_config
-
-    def extract_scores(eval_res: EvalResult) -> dict[str, float | None]:
-        summary_stats = compute_summary_statistics(
-            suite_config=first_suite_config,
-            split=split,
-            results=eval_res.results,
-        )
-
-        values: dict[str, float | None] = {}
-        for key in summary_stats:
-            if key == "overall":
-                values["overall/score"] = summary_stats[key].score
-                values["overall/cost"] = summary_stats[key].cost
-            elif key.startswith("tag/"):
-                tag = key.split("/")[1]
-                values[f"tag/{tag}/score"] = summary_stats[key].score
-                values[f"tag/{tag}/cost"] = summary_stats[key].cost
-            elif key.startswith("task/"):
-                task = key.split("/")[1]
-                values[f"task/{task}/score"] = summary_stats[key].score
-                values[f"task/{task}/score_stderr"] = summary_stats[key].score_stderr
-                values[f"task/{task}/cost"] = summary_stats[key].cost
-                values[f"task/{task}/cost_stderr"] = summary_stats[key].cost_stderr
-        return values
-
-    def format_row(row) -> dict[str, float | str | None]:
-        eval_res = EvalResult.model_validate(row)
-        sub = eval_res.submission
-        sub.submit_time = sub.submit_time or datetime(1970, 1, 1, 0, 0, 0)
-        data = {
-            "submit_time": sub.submit_time.astimezone(ZoneInfo("US/Pacific")).strftime(
-                "%Y-%m-%d"
-            ),
-            "agent_name": (
-                hyperlink(sub.agent_url, sub.agent_name)
-                if sub.agent_url
-                else sub.agent_name
-            ),
-            "agent_description": sub.agent_description or "",
-            "username": sub.username or "",
-            **extract_scores(eval_res),
-            "logs_url": (
-                hyperlink(
-                    hf_uri_to_web_url(
-                        sub.logs_url if IS_INTERNAL else sub.logs_url_public
-                    ),
-                    "🔗",
-                )
-                if (sub.logs_url or sub.logs_url_public)
-                else ""
-            ),
-        }
-        return data
-
-    local_df = local_df.map(format_row)
-
-    df = pd.DataFrame(local_df)
-
-    # Multiply score, cost, and stderr values by 100 and round to 1 decimal
-    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
-    df[numeric_cols] = df[numeric_cols].multiply(100).round(1)
-
-    # Build column order on raw names, then rename via pretty_column_name
-    all_cols = df.columns.tolist()
-    base = ["agent_name", "agent_description", "username"]
-    overall = ["overall/score", "overall/cost"]
-    tags = sorted({c.split("/")[1] for c in all_cols if c.startswith("tag/")})
-    tasks = sorted({c.split("/")[1] for c in all_cols if c.startswith("task/")})
-    rest = ["submit_time", "logs_url"]
-    column_order = (
-        base
-        + overall
-        + [col for tag in tags for col in (f"tag/{tag}/score", f"tag/{tag}/cost")]
-        + [
-            col
-            for t in tasks
-            for col in (
-                f"task/{t}/score",
-                f"task/{t}/score_stderr",
-                f"task/{t}/cost",
-                f"task/{t}/cost_stderr",
-            )
-        ]
-        + rest
-    )
-    df = df.reindex(columns=[c for c in column_order if c in all_cols])
-    # sort by overall score (descending)
-    df = df.sort_values(by=["overall/score"], ascending=False)
-    # apply all renames via pretty_column_name
-    orig_cols = df.columns.tolist()
-    df.columns = [pretty_column_name(col) for col in orig_cols]
-
-    # blank out any null/NaN cells
-    df = df.fillna("")
-
-    return df
-
-
-def load_and_format_dataframes():
-    eval_results = try_load_dataset(
-        RESULTS_DATASET,
-        CONFIG_NAME,
-        download_mode="force_redownload",
-        verification_mode=VerificationMode.NO_CHECKS,
-        trust_remote_code=True,
-    )
-    eval_dataframe_val = get_dataframe_from_results(
-        eval_results=eval_results, split="validation"
-    )
-    eval_dataframe_test = get_dataframe_from_results(
-        eval_results=eval_results, split="test"
-    )
-    return eval_results, eval_dataframe_val, eval_dataframe_test
-
-
-# Display the results
-eval_results, eval_dataframe_val, eval_dataframe_test = load_and_format_dataframes()
-
-
-def restart_space():
-    api.restart_space(repo_id=LEADERBOARD_PATH)
-
-
-def checked_upload_folder(
-    api,
-    folder_path: str,
-    repo_id: str,
-    config_name: str,
-    split: str,
-    submission_name: str,
-) -> str:
-    """Upload with inline size check; raises ValueError if too large."""
-    total = 0
-    for root, _, files in os.walk(folder_path):
-        for f in files:
-            total += os.path.getsize(os.path.join(root, f))
-            if total > MAX_UPLOAD_BYTES:
-                raise ValueError(
-                    f"Upload too large: exceeds {MAX_UPLOAD_BYTES // (1024**2)} MB limit."
-                )
-    # NOTE: This function raises ValueError if unsafe characters are found in the path.
-    return upload_folder_to_hf(
-        api=api,
-        folder_path=folder_path,
-        repo_id=repo_id,
-        config_name=config_name,
-        split=split,
-        submission_name=submission_name,
-    )
-
-
-def add_new_eval(
-    val_or_test: str,
-    agent_name: str | None,
-    agent_description: str,
-    agent_url: str,
-    path_to_file: tempfile._TemporaryFileWrapper | None,
-    username: str,
-    mail: str,
-    profile: gr.OAuthProfile,
-):
-    # default username if none provided
-    if not username or username.strip() == "":
-        username = profile.username
-
-    if not agent_name:
-        return format_warning("Please provide an agent name.")
-
-    submission_time = datetime.now(timezone.utc)
-
-    # Was the profile created less than 2 month ago?
-    user_data = requests.get(
-        f"https://huggingface.co/api/users/{profile.username}/overview"
-    )
-    creation_date = json.loads(user_data.content)["createdAt"]
-
-    created_at = datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%fZ").replace(
-        tzinfo=timezone.utc
-    )
-    if submission_time - created_at < timedelta(days=60):
-        return format_error("This account is not authorized to submit here.")
-
-    contact_infos = try_load_dataset(
-        CONTACT_DATASET,
-        CONFIG_NAME,
-        download_mode="force_redownload",
-        verification_mode=VerificationMode.NO_CHECKS,
-        trust_remote_code=True,
-    )
-    user_submission_dates = sorted(
-        datetime.fromisoformat(row["submit_time"])
-        for row in contact_infos.get(val_or_test, [])
-        if row["username_auth"] == profile.username
-    )
-    if len(user_submission_dates) > 0 and abs(
-        submission_time - user_submission_dates[-1]
-    ) < timedelta(seconds=24 * 60 * 60):
-        return format_error(
-            "You already submitted once in the last 24h; please try again later."
-        )
-
-    is_validation = val_or_test == "validation"
-
-    # Very basic email parsing
-    _, parsed_mail = parseaddr(mail)
-    if "@" not in parsed_mail:
-        return format_warning("Please provide a valid email adress.")
-
-    # Check duplicate submissions by inspecting the nested "submission" dicts
-    if val_or_test in eval_results and len(eval_results[val_or_test]) > 0:
-        existing = eval_results[val_or_test]
-        subs = existing.to_dict().get("submission", [])
-        names = {item.get("agent_name", "").lower() for item in subs}
-        users = {item.get("username", "").lower() for item in subs}
-        if agent_name.lower() in names and username.lower() in users:
-            return format_warning("This agent has been already submitted.")
-
-    if path_to_file is None:
-        return format_warning("Please attach a file.")
-
-    # sanitize username and agent_name for filesystem
-    safe_username = sanitize_path_component(username)
-    safe_agent_name = sanitize_path_component(agent_name)
-
-    extracted_dir = os.path.join(
-        EXTRACTED_DATA_DIR, f"{safe_username}_{safe_agent_name}"
-    )
-
-    if LOCAL_DEBUG:
-        print("mock extracted file", flush=True)
-    else:
-        try:
-            # 1) remove old extraction if present
-            if os.path.exists(extracted_dir):
-                shutil.rmtree(extracted_dir)
-            os.makedirs(extracted_dir, exist_ok=True)
-
-            # 2) securely extract only regular files, flatten structure
-            # Flatten structure to aid finding the manifest agenteval.json file
-            # and because hierarchical structure is not needed
-            with tarfile.open(path_to_file.name, "r:gz") as tar:
-                for member in tar.getmembers():
-                    if not member.isreg():
-                        continue
-                    fname = os.path.basename(member.name)
-                    # skip empty or hidden
-                    if not fname or fname.startswith("."):
-                        continue
-                    fobj = tar.extractfile(member)
-                    if not fobj:
-                        continue
-                    target = os.path.join(extracted_dir, fname)
-                    with open(target, "wb") as out:
-                        out.write(fobj.read())
-
-            # 3) ensure something was extracted
-            if not os.listdir(extracted_dir):
-                return format_error("Submission tarball is empty or invalid.")
-
-        except Exception as e:
-            return format_error(
-                f"Error while extracting the file: {e}. Be sure to upload a valid .tar.gz file."
-            )
-
-    submission_name = (
-        f"{safe_username}_{safe_agent_name}_{submission_time.strftime('%Y-%m-%d')}"
+LOGO_PATH = "Ai2_logo_pink_padding_RGB.png"
+
+
+
+# --- Theme Definition ---
+theme = gr.themes.Base(
+    primary_hue=gr.themes.Color(c100="#CFF5E8", c200="#B7EFDD", c300="#9FEAD1", c400="#87E5C5", c50="#E7FAF3", c500="#6FE0BA", c600="#57DBAF", c700="#3FD5A3", c800="#27D09C", c900="#0FCB8C", c950="#0fcb8c"),
+    secondary_hue=gr.themes.Color(c100="#FCDCEB", c200="#FBCBE1", c300="#F9BAD7", c400="#F7A8CD", c50="#FDEEF5", c500="#F697C4", c600="#F586BA", c700="#F375B0", c800="#F263A6", c900="#F0529C", c950="#F0529C"),
+    neutral_hue=gr.themes.Color(c100="#FDF9F4", c200="#C9C9C3", c300="#B0B5AF", c400="#97A09C", c50="#FAF2E9", c500="#7F8C89", c600="#667876", c700="#344F4F", c800="#1C3A3C", c900="#032629", c950="032629"),
+    font=[gr.themes.GoogleFont('Manrope'), 'ui-sans-serif', 'sans-serif', 'sans-serif'],
+    font_mono=[gr.themes.GoogleFont('Roboto Mono'), 'ui-monospace', 'monospace', 'monospace'],
+).set(
+    body_text_color='*neutral_950',
+    body_text_color_dark='*neutral_50',
+    background_fill_primary='*neutral_50',
+    background_fill_primary_dark='*neutral_900',
+    background_fill_secondary='*neutral_100',
+    background_fill_secondary_dark='*neutral_800',
+    border_color_accent='*secondary_900',
+    border_color_accent_subdued='*neutral_400',
+    border_color_accent_subdued_dark='*neutral_400',
+    color_accent='*primary_900',
+    color_accent_soft='*neutral_200',
+    color_accent_soft_dark='*neutral_800',
+    link_text_color='*secondary_900',
+    link_text_color_dark='*primary_900',
+    link_text_color_active_dark='*primary_600',
+    link_text_color_hover_dark='*primary_700',
+    link_text_color_visited_dark='*primary_600',
+    table_even_background_fill='*neutral_100',
+    table_even_background_fill_dark='*neutral_800',
+    button_primary_background_fill='*secondary_900',
+    button_primary_background_fill_dark='*primary_900',
+    button_primary_background_fill_hover='*secondary_600',
+    button_primary_background_fill_hover_dark='*primary_600',
+    button_primary_text_color='*neutral_900',
+    button_primary_text_color_dark='*neutral_900'
+)
+# --- Gradio App Definition ---
+demo = gr.Blocks(theme=theme, css=css)
+with demo:
+    gr.Image(
+        value=LOGO_PATH,
+        show_label=False,
+        interactive=False,
+        container=False,
+        show_download_button=False,
+        show_fullscreen_button=False,
+        elem_id="logo-image"
     )
+    gr.HTML(TITLE)
 
-    # SAVE UNSCORED SUBMISSION
-    if LOCAL_DEBUG:
-        print("mock uploaded submission", flush=True)
-    else:
-        try:
-            checked_upload_folder(
-                api=api,
-                folder_path=extracted_dir,
-                repo_id=SUBMISSION_DATASET,
-                config_name=CONFIG_NAME,
-                split=val_or_test,
-                submission_name=submission_name,
-            )
-        except ValueError as e:
-            return format_error(str(e))
-
-    # SAVE CONTACT
-    contact_info = {
-        "agent_name": agent_name,
-        "agent_description": agent_description,
-        "url": agent_url,
-        "username": username,
-        "username_auth": profile.username,
-        "mail": mail,
-        "submit_time": submission_time.isoformat(),
-    }
-    # add or init contact dataset for this split
-    if val_or_test in contact_infos:
-        contact_infos[val_or_test] = contact_infos[val_or_test].add_item(contact_info)
-    else:
-        contact_infos[val_or_test] = Dataset.from_list([contact_info])
-    if LOCAL_DEBUG:
-        print("mock uploaded contact info", flush=True)
-    else:
-        contact_infos.push_to_hub(CONTACT_DATASET, config_name=CONFIG_NAME)
-
+    main_page.demo.render()
+with demo.route("Literature Understanding"):
+    literature_understanding.demo.render()
+with demo.route("Code & Execution"):
+    c_and_e.demo.render()
+with demo.route("Data Analysis"):
+    data_analysis.demo.render()
+with demo.route("Discovery"):
+    e2e.demo.render()
+
+# --- Scheduler and Launch
+def restart_space_job():
+    print("Scheduler: Attempting to restart space.")
     try:
-        json_path = Path(extracted_dir) / AGENTEVAL_MANIFEST_NAME
-        if not json_path.exists():
-            return format_error(f"Missing manifest {AGENTEVAL_MANIFEST_NAME}")
-        raw = json_path.read_text(encoding="utf-8")
-        eval_result = EvalResult.model_validate_json(raw)
-        if eval_result.suite_config.version != CONFIG_NAME:
-            return format_error(
-                f"Error: submitted suite version {eval_result.suite_config.version} "
-                f"does not match currently accepted version {CONFIG_NAME}"
-            )
-        if eval_result.split != val_or_test:
-            return format_error(
-                f"Error: uploaded split {eval_result.split} does not match selected split {val_or_test}"
-            )
-
-        # NOTE: Trusting user-computed scores, but re-computing the derived results based on the log files
-        eval_result.results = process_eval_logs(extracted_dir)[0]
-        eval_result.save_json(str(json_path))
-
+        api.restart_space(repo_id=LEADERBOARD_PATH)
+        print("Scheduler: Space restart request sent.")
     except Exception as e:
-        return format_error(
-            f"Error while scoring the submission: {e}. Be sure to upload a valid submission."
-        )
+        print(f"Scheduler: Error restarting space: {e}")
+    scheduler = BackgroundScheduler(timezone="UTC")
+    scheduler.add_job(restart_space_job, "interval", hours=1)
+    scheduler.start()
 
-    # # SAVE SCORED SUBMISSION
-    if LOCAL_DEBUG:
-        print("mock uploaded scored submission")
-    else:
-        try:
-            logs_url_private = checked_upload_folder(
-                api=api,
-                folder_path=extracted_dir,
-                repo_id=SUBMISSION_DATASET,
-                config_name=CONFIG_NAME,
-                split=val_or_test,
-                submission_name=f"{submission_name}_scored",
-            )
-        except ValueError as e:
-            return format_error(str(e))
-
-        # Validation submissions are public for public leaderboard
-        if is_validation and not IS_INTERNAL:
-            try:
-                logs_url_public = checked_upload_folder(
-                    api=api,
-                    folder_path=extracted_dir,
-                    repo_id=SUBMISSION_DATASET_PUBLIC,
-                    config_name=CONFIG_NAME,
-                    split=val_or_test,
-                    submission_name=f"{submission_name}_scored",
-                )
-            except ValueError as e:
-                return format_error(str(e))
-        else:
-            logs_url_public = None
-
-    eval_result.submission.agent_name = agent_name
-    eval_result.submission.agent_description = agent_description
-    eval_result.submission.agent_url = agent_url
-    eval_result.submission.username = username
-    eval_result.submission.submit_time = submission_time
-    eval_result.submission.logs_url = logs_url_private
-    eval_result.submission.logs_url_public = logs_url_public
 
+# Launch the Gradio app
+if __name__ == "__main__":
     if LOCAL_DEBUG:
-        print("mock uploaded results to lb")
+        print("Launching in LOCAL_DEBUG mode.")
+        def get_initial_global_tag_choices(): return ["Overall", "TagA"]
+        demo.launch(debug=True)
     else:
-        upload_summary_to_hf(
-            api=api,
-            eval_result=eval_result,
-            repo_id=RESULTS_DATASET,
-            config_name=CONFIG_NAME,
-            split=val_or_test,
-            submission_name=f"{submission_name}_scored",
-        )
-
-    return format_log(
-        f"Agent {agent_name} submitted by {username} successfully.\nPlease wait a few hours and refresh the leaderboard to see your score displayed."
-    )
-
-
-def refresh():
-    _, eval_dataframe_val, eval_dataframe_test = load_and_format_dataframes()
-    return eval_dataframe_val, eval_dataframe_test
-
-
-# Determine column types dynamically based on dataframe columns
-def compute_column_types(df):
-    col_types = []
-    for col in df.columns:
-        if col == "Agent":
-            col_types.append("markdown")
-        elif col in ["Agent description", "User/organization", "Submission date"]:
-            col_types.append("str")
-        elif col == "Logs":
-            col_types.append("markdown")
-        else:
-            col_types.append("number")
-    return col_types
-
-
-test_col_types = compute_column_types(eval_dataframe_test)
-val_col_types = compute_column_types(eval_dataframe_val)
-
-demo = gr.Blocks()
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                elem_id="citation-button",
-            )  # .style(show_copy_button=True)
-
-    leaderboard_table_test = gr.Dataframe(
-        value=eval_dataframe_test,
-        headers=list(eval_dataframe_test.columns),
-        datatype=test_col_types,
-        interactive=False,
-        column_widths=["20%"],
-        render=False,
-    )
-
-    leaderboard_table_val = gr.Dataframe(
-        value=eval_dataframe_val,
-        headers=list(eval_dataframe_val.columns),
-        datatype=val_col_types,
-        interactive=False,
-        column_widths=["20%"],
-        render=False,
-    )
-
-    # Build tab layout list based on desired order
-    tabs = [
-        ("Results: Test", leaderboard_table_test),
-        ("Results: Validation", leaderboard_table_val),
-    ]
-
-    if IS_INTERNAL:
-        tabs = [tabs[1], tabs[0]]  # Validation first for internal users
-
-    # Render the tabs in desired order
-    for label, component in tabs:
-        with gr.Tab(label):
-            component.render()
-
-    refresh_button = gr.Button("Refresh")
-    refresh_button.click(
-        refresh,
-        inputs=[],
-        outputs=[
-            leaderboard_table_val,
-            leaderboard_table_test,
-        ],
-    )
-    with gr.Accordion("Submit a new agent for evaluation"):
-        with gr.Row():
-            gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
-        with gr.Row():
-            with gr.Column():
-                level_of_test = gr.Radio(
-                    ["validation", "test"], value="validation", label="Split"
-                )
-                agent_name_textbox = gr.Textbox(label="Agent name")
-                agent_description_textbox = gr.Textbox(label="Agent description")
-                agent_url_textbox = gr.Textbox(label="Url to agent information")
-            with gr.Column():
-                username = gr.Textbox(
-                    label="Organization or user name (defaults to your HF username)",
-                    placeholder="Leave blank to use your HF username",
-                )
-                mail = gr.Textbox(
-                    label="Contact email (will be stored privately, & used if there is an issue with your submission)"
-                )
-                file_output = gr.File()
-
-        with gr.Row():
-            gr.LoginButton()
-            submit_button = gr.Button("Submit Eval")
-        submission_result = gr.Markdown()
-        submit_button.click(
-            add_new_eval,
-            [
-                level_of_test,
-                agent_name_textbox,
-                agent_description_textbox,
-                agent_url_textbox,
-                file_output,
-                username,
-                mail,
-            ],
-            submission_result,
-        )
+        print("Launching in Space mode.")
+        # For Spaces, share=False is typical unless specific tunneling is needed.
+        # debug=True can be set to False for a "production" Space.
+        demo.launch(server_name="0.0.0.0", server_port=7860, debug=True, share=False)
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=3600)
-scheduler.start()
-if LOCAL_DEBUG:
-    demo.launch(debug=True)
-else:
-    demo.launch(server_name="0.0.0.0", server_port=7860, debug=True)

c_and_e.py

@@ -0,0 +1,54 @@
+import gradio as gr
+import pandas as pd
+
+# Import our UI factories and the data loader
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
+
+# Define the category for this page
+CATEGORY_NAME = "Code Execution"
+
+with gr.Blocks() as demo:
+    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+
+    # --- This page now has two main sections: Validation and Test ---
+    with gr.Tabs():
+        with gr.Tab("Results: Validation"):
+            # 1. Load all necessary data for the "validation" split ONCE.
+            validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+
+            if not validation_df.empty:
+                # 2. Render the main category display using the loaded data.
+                create_leaderboard_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="validation"
+                )
+
+                # 3. Render the detailed breakdown for each benchmark in the category.
+                create_benchmark_details_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for validation split.")
+
+        with gr.Tab("Results: Test"):
+            # Repeat the process for the "test" split
+            test_df, test_tag_map = get_full_leaderboard_data("test")
+
+            if not test_df.empty:
+                create_leaderboard_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="test"
+                )
+                create_benchmark_details_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for test split.")

content.py

@@ -3,6 +3,14 @@ TITLE = """<h1 align="center" id="space-title">AstaBench Leaderboard</h1>"""
 INTRODUCTION_TEXT = """
 ## Introduction
 """
+INTRO_PARAGRAPH = """
+AI agents are on the rise, promising everything from travel planning to scientific discovery. But evaluating them—especially for real-world research tasks—remains a messy, inconsistent process. Metrics vary, cost is often ignored, and scientific use cases are rarely the focus. <br>
+<br>
+Enter AstaBench, a grand challenge benchmark developed by Ai2 to test how well agentic AI systems perform on scientific tasks that actually matter. As part of the Asta initiative, AstaBench spans ten multi-step benchmarks covering literature review, data analysis, code execution, and complex decision-making. It brings standardization and transparency to agent evaluation, with statistical confidence reporting, and a leaderboard that highlights tradeoffs between accuracy and computational cost.
+"""
+SCATTER_DISCLAIMER = """
+Only agents that have cost data available will be shown in the scatter plot. If you don't see your agent, please ensure that you have provided cost data in your submission.
+"""
 
 SUBMISSION_TEXT = """
 ## Submissions
@@ -32,8 +40,11 @@ def format_log(msg):
     return f"<p style='color: green; font-size: 20px; text-align: center;'>{msg}</p>"
 
 
-def hyperlink(link, text):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{text}</a>'
+def hyperlink(link_url: str, text: str = "🔗") -> str:
+    if not link_url or not isinstance(link_url, str):
+        return str(text) # Or simply "" if link_url is bad
+    # Using a simpler style here for broad compatibility, your original style is fine too.
+    return f'<a target="_blank" href="{link_url}">{text}</a>'
 
 
 def hf_uri_to_web_url(uri: str) -> str:
@@ -53,3 +64,56 @@ def hf_uri_to_web_url(uri: str) -> str:
 
     namespace, repo, path = parts
     return f"https://huggingface.co/datasets/{namespace}/{repo}/tree/main/{path}"
+
+css = """
+.submission-accordion {
+    border-style: solid;
+    border-width: 3px !important;
+    border-color: #ec4899;
+}
+.submission-accordion span.svelte-1w6vloh { 
+    font-weight: bold !important;
+    font-size: 1.2em !important;
+}
+#logo-image { 
+    margin: auto;          
+    max-width: 250px;       
+    height: auto;           
+}
+.table-component{
+    height: auto !important;
+    max-height: none !important;
+}
+
+.table-wrap {
+    max-height: none !important;
+    height: auto !important;
+    overflow-y: visible !important;
+}
+/* --- New Rules for Table Density --- */
+table.gr-table th, table.gr-table td {
+    padding: 4px 4px !important; 
+    width: 1%;
+    white-space: nowrap;
+}
+
+table.gr-table {
+    font-size: 14px !important;
+}
+
+/* Example of making the "Agent" column (the 1st column) a bit wider if needed */
+table.gr-table th:nth-child(1),
+table.gr-table td:nth-child(1) {
+    min-width: 150px !important;
+    white-space: normal !important; /* Allow agent names to wrap if long */
+}
+.html-container {
+    padding-top: 0 !important;
+}
+#scatter-disclaimer {
+    color: #f0529c !important;
+}
+thead.svelte-1e98i6s th {
+    background: white !important;
+}
+"""

data/1.0.0-dev1/agenteval.json

@@ -0,0 +1,332 @@
+{
+  "suite_config": {
+    "name": "asta-bench",
+    "version": "1.0.0-dev1",
+    "splits": [
+      {
+        "name": "validation",
+        "tasks": [
+          {
+            "name": "arxivdigestables_validation",
+            "path": "astabench/arxivdigestables_validation",
+            "primary_metric": "score_tables/mean",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "sqa_dev",
+            "path": "astabench/sqa_dev",
+            "primary_metric": "global_avg/mean",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "litqa2_validation",
+            "path": "astabench/litqa2_validation",
+            "primary_metric": "is_correct/accuracy",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "paper_finder_validation",
+            "path": "astabench/paper_finder_validation",
+            "primary_metric": "score_paper_finder/macro_avg",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "discoverybench_validation",
+            "path": "astabench/discoverybench_validation",
+            "primary_metric": "score_discoverybench/mean",
+            "tags": [
+              "data"
+            ]
+          },
+          {
+            "name": "core_bench_validation",
+            "path": "astabench/core_bench_validation",
+            "primary_metric": "evaluate_task_questions/accuracy",
+            "tags": [
+              "code"
+            ]
+          },
+          {
+            "name": "ds1000_validation",
+            "path": "astabench/ds1000_validation",
+            "primary_metric": "ds1000_scorer/accuracy",
+            "tags": [
+              "code"
+            ]
+          },
+          {
+            "name": "e2e_discovery_validation",
+            "path": "astabench/e2e_discovery_validation",
+            "primary_metric": "score_rubric/accuracy",
+            "tags": [
+              "discovery"
+            ]
+          },
+          {
+            "name": "super_validation",
+            "path": "astabench/super_validation",
+            "primary_metric": "check_super_execution/entrypoints",
+            "tags": [
+              "code"
+            ]
+          }
+        ]
+      },
+      {
+        "name": "test",
+        "tasks": [
+          {
+            "name": "paper_finder_test",
+            "path": "astabench/paper_finder_test",
+            "primary_metric": "score_paper_finder/macro_avg",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "sqa_test",
+            "path": "astabench/sqa_test",
+            "primary_metric": "global_avg/mean",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "arxivdigestables_test",
+            "path": "astabench/arxivdigestables_test",
+            "primary_metric": "score_tables/mean",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "litqa2_test",
+            "path": "astabench/litqa2_test",
+            "primary_metric": "is_correct/accuracy",
+            "tags": [
+              "lit"
+            ]
+          },
+          {
+            "name": "discoverybench_test",
+            "path": "astabench/discoverybench_test",
+            "primary_metric": "score_discoverybench/mean",
+            "tags": [
+              "data"
+            ]
+          },
+          {
+            "name": "core_bench_test",
+            "path": "astabench/core_bench_test",
+            "primary_metric": "evaluate_task_questions/accuracy",
+            "tags": [
+              "code"
+            ]
+          },
+          {
+            "name": "ds1000_test",
+            "path": "astabench/ds1000_test",
+            "primary_metric": "ds1000_scorer/accuracy",
+            "tags": [
+              "code"
+            ]
+          },
+          {
+            "name": "e2e_discovery_test",
+            "path": "astabench/e2e_discovery_test",
+            "primary_metric": "score_rubric/accuracy",
+            "tags": [
+              "discovery"
+            ]
+          },
+          {
+            "name": "super_test",
+            "path": "astabench/super_test",
+            "primary_metric": "check_super_execution/entrypoints",
+            "tags": [
+              "code"
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "split": "validation",
+  "results": [
+    {
+      "task_name": "sqa_dev",
+      "metrics": [
+        {
+          "name": "global_avg/mean",
+          "value": 0.6215245045241414
+        },
+        {
+          "name": "global_avg/stderr",
+          "value": 0.02088486499225903
+        },
+        {
+          "name": "ingredient_recall/mean",
+          "value": 0.6029178145087237
+        },
+        {
+          "name": "ingredient_recall/stderr",
+          "value": 0.026215888361291618
+        },
+        {
+          "name": "answer_precision/mean",
+          "value": 0.7960436785436785
+        },
+        {
+          "name": "answer_precision/stderr",
+          "value": 0.027692773517249983
+        },
+        {
+          "name": "citation_precision/mean",
+          "value": 0.697849041353826
+        },
+        {
+          "name": "citation_precision/stderr",
+          "value": 0.026784164936602798
+        },
+        {
+          "name": "citation_recall/mean",
+          "value": 0.3892874836903378
+        },
+        {
+          "name": "citation_recall/stderr",
+          "value": 0.015094770200171756
+        }
+      ],
+      "model_costs": [
+        1.3829150000000001,
+        0.9759700000000001,
+        2.2324650000000004,
+        0.76631,
+        0.9277900000000001,
+        2.6388600000000006,
+        0.8114100000000002,
+        2.3263174999999996,
+        2.5423725,
+        1.2398675000000001,
+        1.7387300000000003,
+        1.2176599999999997,
+        0.564655,
+        0.9726750000000001,
+        0.7675700000000001,
+        1.5198850000000002,
+        1.4726625000000002,
+        2.1937650000000004,
+        0.6907700000000001,
+        1.39835,
+        1.2598175,
+        2.5373550000000002,
+        2.19239,
+        1.2508875000000006,
+        2.2650550000000007,
+        1.6047725,
+        0.6525125000000003,
+        1.4262200000000003,
+        1.0533299999999999,
+        1.7252375,
+        1.407145,
+        1.5408700000000004,
+        2.8073224999999993,
+        1.0448125000000006,
+        1.7037300000000004,
+        0.8650500000000001,
+        1.0171225000000002,
+        0.5697925000000001,
+        2.7851025,
+        1.0551425,
+        2.9213775,
+        1.7772975000000004,
+        1.2753225000000001,
+        0.8108325000000001,
+        0.6958375000000001,
+        0.8840950000000003,
+        1.2028724999999998,
+        1.2490475000000003,
+        2.4272,
+        1.95026,
+        1.5352475,
+        2.11181,
+        2.3612249999999997,
+        1.8619225000000004,
+        0.7431075000000001,
+        1.5189675000000002,
+        1.089575,
+        1.6103700000000003,
+        1.4201450000000002,
+        2.397835,
+        1.469175,
+        1.0723550000000004,
+        0.7964050000000003,
+        3.3733175,
+        4.197085,
+        4.2637675,
+        1.2982124999999998,
+        0.66146,
+        1.1130475000000002,
+        2.4393974999999997,
+        2.582,
+        1.7381725000000001,
+        0.415025,
+        1.6777325,
+        1.0507825000000002,
+        2.4627125000000003,
+        1.017005,
+        1.9210250000000002,
+        1.5009025000000003,
+        0.8283125000000001,
+        2.9854425,
+        0.4633375000000001,
+        0.397685,
+        1.2803425,
+        3.0388200000000003,
+        1.2610875000000004,
+        1.798365,
+        3.427287500000001,
+        0.29307750000000005,
+        0.37101249999999997,
+        2.8046925000000003,
+        0.35557000000000005,
+        3.5481700000000007,
+        1.1073975,
+        1.5280825,
+        1.1714900000000001,
+        3.1791275000000003,
+        3.8214725000000005,
+        1.8440275,
+        1.730515,
+        1.9350675000000002,
+        1.6592125000000002,
+        1.9227124999999998,
+        1.202885,
+        1.2688150000000002,
+        0.8819875000000001,
+        0.6989325,
+        1.965635,
+        1.7467800000000002,
+        1.6940625000000002
+      ]
+    }
+  ],
+  "submission": {
+    "submit_time": "2025-06-09T20:55:35.869831Z",
+    "username": "miked-ai",
+    "agent_name": "Basic ReAct",
+    "agent_description": null,
+    "agent_url": null,
+    "logs_url": "hf://datasets/allenai/asta-bench-internal-submissions/1.0.0-dev1/validation/miked-ai_Basic_ReAct__task_tools__report_editor__2025-06-09T20-55-35",
+    "logs_url_public": null,
+    "summary_url": null
+  }
+}

data_analysis.py

@@ -0,0 +1,54 @@
+import gradio as gr
+import pandas as pd
+
+# Import our UI factories and the data loader
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
+
+# Define the category for this page
+CATEGORY_NAME = "Data Analysis"
+
+with gr.Blocks() as demo:
+    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+
+    # --- This page now has two main sections: Validation and Test ---
+    with gr.Tabs():
+        with gr.Tab("Results: Validation"):
+            # 1. Load all necessary data for the "validation" split ONCE.
+            validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+
+            if not validation_df.empty:
+                # 2. Render the main category display using the loaded data.
+                create_leaderboard_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="validation"
+                )
+
+                # 3. Render the detailed breakdown for each benchmark in the category.
+                create_benchmark_details_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for validation split.")
+
+        with gr.Tab("Results: Test"):
+            # Repeat the process for the "test" split
+            test_df, test_tag_map = get_full_leaderboard_data("test")
+
+            if not test_df.empty:
+                create_leaderboard_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="test"
+                )
+                create_benchmark_details_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for test split.")

e2e.py

@@ -0,0 +1,54 @@
+import gradio as gr
+import pandas as pd
+
+# Import our UI factories and the data loader
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
+
+# Define the category for this page
+CATEGORY_NAME = "Discovery"
+
+with gr.Blocks() as demo:
+    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+
+    # --- This page now has two main sections: Validation and Test ---
+    with gr.Tabs():
+        with gr.Tab("Results: Validation"):
+            # 1. Load all necessary data for the "validation" split ONCE.
+            validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+
+            if not validation_df.empty:
+                # 2. Render the main category display using the loaded data.
+                create_leaderboard_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="validation"
+                )
+
+                # 3. Render the detailed breakdown for each benchmark in the category.
+                create_benchmark_details_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for validation split.")
+
+        with gr.Tab("Results: Test"):
+            # Repeat the process for the "test" split
+            test_df, test_tag_map = get_full_leaderboard_data("test")
+
+            if not test_df.empty:
+                create_leaderboard_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="test"
+                )
+                create_benchmark_details_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for test split.")

json_leaderboard.py

@@ -0,0 +1,485 @@
+import logging
+from typing import Optional, Any, Dict # Added Dict
+from zoneinfo import ZoneInfo
+
+# datasets import might not be strictly needed by LeaderboardViewer itself anymore,
+# but _get_dataframe might still use types from it if EvalResult refers to them.
+# For now, let's keep it if your EvalResult or SuiteConfig models have dependencies.
+# If not, it can be removed from here.
+import datasets # Potentially removable from this file
+import matplotlib.pyplot as plt
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+import pandas as pd
+import seaborn as sns
+import json # For loading the local JSON file
+import os # For checking file existence
+
+from agenteval import compute_summary_statistics
+from agenteval.config import SuiteConfig
+from agenteval.models import EvalResult
+
+logger = logging.getLogger(__name__)
+
+import logging
+from typing import Optional, Any, Dict, List # Added List
+from zoneinfo import ZoneInfo # Assuming this might be used by SuiteConfig/EvalResult or _get_dataframe
+import json
+import os
+
+# Assuming these are correctly imported from your project
+from agenteval.config import SuiteConfig
+from agenteval.models import EvalResult
+# from agenteval import compute_summary_statistics # Used by _get_dataframe
+
+
+class DataTransformer:
+    """
+    Load and visualize leaderboard from a single, local JSON result file.
+    """
+    _INFORMAL_TO_FORMAL_NAME_MAP = {
+        "lit": "Literature Understanding",
+        "data": "Data Analysis",
+        "code": "Code Execution",
+        "discovery": "Discovery",
+        "arxivdigestables_validation": "Arxivdigestables Validation",
+        "sqa_dev": "Sqa Dev",
+        "litqa2_validation": "Litqa2 Validation",
+        "paper_finder_validation": "Paper Finder Validation",
+        "discoverybench_validation": "Discoverybench Validation",
+        "core_bench_validation": "Core Bench Validation",
+        "ds1000_validation": "DS1000 Validation",
+        "e2e_discovery_validation": "E2E Discovery Validation",
+        "super_validation": "Super Validation",
+        # Add any other raw names that can appear in task.name or task.tags
+    }
+
+    def __init__(
+            self,
+            json_file_path: str, # Mandatory: path to the local JSON file
+            split: str,          # Still needed for context within the JSON's suite_config
+            is_internal: bool = False
+    ):
+        self._json_file_path = json_file_path
+        self._split = split
+        self._internal = is_internal
+        self._loaded_json_data: Optional[Dict[str, Any]] = None
+        self._cfg: Optional[SuiteConfig] = None
+
+        logger.info(f"Initializing LeaderboardViewer with local JSON file: {self._json_file_path}")
+
+        # --- Load and Validate JSON data ---
+        if not os.path.exists(self._json_file_path):
+            raise FileNotFoundError(f"JSON file not found at path: {self._json_file_path}")
+        try:
+            with open(self._json_file_path, 'r', encoding='utf-8') as f:
+                self._loaded_json_data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Failed to parse JSON from local file {self._json_file_path}: {e}")
+        except Exception as e:
+            raise ValueError(f"Error reading local file {self._json_file_path}: {e}")
+
+        if not self._loaded_json_data:
+            raise ValueError(f"No data loaded from JSON file {self._json_file_path}.")
+
+        try:
+            eval_result = EvalResult.model_validate(self._loaded_json_data)
+        except Exception as e:
+            raise ValueError(f"Failed to validate JSON data from file '{self._json_file_path}' against EvalResult model: {e}")
+
+        self._cfg = eval_result.suite_config
+        if not isinstance(self._cfg, SuiteConfig):
+            raise TypeError(f"self._cfg is not a SuiteConfig object after loading from '{self._json_file_path}', got {type(self._cfg)}.")
+
+        # --- Populate Tag Map (Corrected Placement and Helper Function Access) ---
+        self.tag_map: dict[str, list[str]] = {}
+
+        # Access tasks from the loaded config
+        tasks_for_split: List[Any] = self._cfg.get_tasks(self._split) # Assuming get_tasks returns a list of task-like objects
+
+        for task in tasks_for_split:
+            # Ensure task object has 'name' and 'tags' attributes
+            if not hasattr(task, 'name') or not hasattr(task, 'tags'):
+                logger.warning(f"Task object {task} is missing 'name' or 'tags' attribute. Skipping.")
+                continue
+
+            formal_task_display_name = self._get_formal_display_name_static(task.name) # Use the helper method
+
+            if not (task.tags or []):
+                continue
+
+            for raw_tag_name in task.tags:
+                formal_tag_display_name_key = self._get_formal_display_name_static(raw_tag_name)
+
+                self.tag_map.setdefault(formal_tag_display_name_key, []).append(formal_task_display_name)
+
+        for key in self.tag_map:
+            self.tag_map[key] = sorted(list(set(self.tag_map[key])))
+
+    # --- Helper function defined as a static method or regular method ---
+    # Option 1: Static method (doesn't need 'self', uses the class attribute)
+    @staticmethod
+    def _get_formal_display_name_static(raw_name: str) -> str:
+        """
+        Helper function to get the formal display name for a raw tag or task name.
+        Uses the class's map and provides a fallback.
+        """
+        return DataTransformer._INFORMAL_TO_FORMAL_NAME_MAP.get(raw_name, raw_name.replace("_", " ").title())
+
+    def _load(self) -> tuple[pd.DataFrame, dict[str, list[str]]]:
+        """
+        Prepares the DataFrame from the loaded JSON data.
+        The JSON data is already loaded and validated in __init__.
+        """
+        if self._loaded_json_data is None or self._cfg is None:
+            # This should not happen if __init__ completed successfully
+            raise RuntimeError("LeaderboardViewer2 not properly initialized. JSON data or SuiteConfig is missing.")
+
+        # The _get_dataframe function expects a list of records.
+        # Since we have a single JSON file representing one result, wrap it in a list.
+        records_list: list[dict] = [self._loaded_json_data]
+
+        overview_df = _get_dataframe(
+            records_list=records_list,
+            split=self._split,
+            is_internal=self._internal,
+            suite_config=self._cfg, # Pass the SuiteConfig loaded in __init__
+        )
+        return overview_df, self.tag_map
+
+    # --- view method remains the same as your last version ---
+    def view(
+            self,
+            tag: Optional[str] = None,
+            with_plots: bool = False,
+            use_plotly: bool = False,
+    ) -> tuple[pd.DataFrame, dict[str, Any]]:
+        data, tag_map = self._load() # tag_map is also returned by _load now
+        print(f"AHAHASHJDBFGASJHDBJAHSDB,AHDB {tag_map}")
+        print(f"THIS IS THE DATA DATA DTAA {data.columns}")
+        if data.empty or (len(data) == 1 and data.iloc[0].get("Agent") == "No data"):
+            logger.warning("No data available to view. Returning empty DataFrame and plots.")
+            return data, {}
+
+        base_cols = ["Agent", "Submitter", "Date", "Logs"]
+        existing_cols = [col for col in base_cols if col in data.columns]
+
+        primary_score_col: str
+        group_metric_names: list[str]
+
+        if tag is None:
+            primary = "Overall"
+            group = list(tag_map.keys())
+        else:
+            primary = tag
+            group = tag_map.get(tag, [])
+
+        if f"{primary} Score" in data.columns:
+            data = data.sort_values(f"{primary} Score", ascending=False)
+        else:
+            logger.warning(f"Primary metric '{primary}' for sorting not found. Data will not be sorted by it.")
+
+        metrics_to_display = []
+        if f"{primary} Cost" in data.columns:
+            metrics_to_display.append(f"{primary} Cost")
+        if f"{primary} Score" in data.columns:
+            metrics_to_display.append(f"{primary} Score")
+
+        for g_item in group:
+            if g_item in data.columns:
+                metrics_to_display.append(g_item)
+            if f"{g_item} Cost" in data.columns:
+                metrics_to_display.append(f"{g_item} Cost")
+            if f"{g_item} Score" in data.columns:
+                metrics_to_display.append(f"{g_item} Score")
+
+
+        final_cols_to_display = existing_cols + [m for m in metrics_to_display if m in data.columns]
+        final_cols_to_display = sorted(list(set(final_cols_to_display)), key=final_cols_to_display.index)
+
+        df_view = data.loc[:, final_cols_to_display].reset_index(drop=True)
+
+        plots: dict[str, Any] = {}
+        if with_plots:
+            plot_metric_names = [primary] + [g_item for g_item in group if g_item in data.columns]
+            for metric_name in plot_metric_names:
+                score_col = f"{metric_name} Score"
+                cost_col = f"{metric_name} Cost"
+                if score_col in df_view.columns and cost_col in df_view.columns:
+                    if use_plotly:
+                        fig = _plot_scatter_plotly(df_view, x=cost_col, y=score_col, agent_col="Agent")
+                    plots[f"scatter_{metric_name}"] = fig
+                else:
+                    logger.warning(
+                        f"Skipping plot for '{metric_name}': score column '{score_col}' or cost column '{cost_col}' not found."
+                    )
+        return df_view, plots
+
+
+def _safe_round(value, digits=2):
+    return round(value, digits) if isinstance(value, (float, int)) and pd.notna(value) else value
+
+def _get_dataframe(
+        records_list: list[dict],
+        split: str,
+        is_internal: bool,
+        suite_config: SuiteConfig,
+        timezone: str = "US/Pacific",
+) -> pd.DataFrame:
+    # This function remains the same as in the previous version you provided.
+    # It takes a list of records (which will be a list containing one item
+    # from the loaded JSON file) and processes it.
+    if not records_list:
+        logger.warning(f"No records provided to _get_dataframe for split '{split}'. Returning empty DataFrame with placeholder.")
+        expected_pretty_cols = ["Agent Name", "Submitter", "Date", "Overall Score", "Logs"]
+        empty_df = pd.DataFrame({p_col: ["No data"] for p_col in expected_pretty_cols})
+        return empty_df
+
+    cfg = suite_config
+
+    rows = []
+    for itm_idx, itm in enumerate(records_list):
+        if not isinstance(itm, dict):
+            logger.warning(f"Item {itm_idx} in records_list is not a dict, skipping.")
+            continue
+        try:
+            ev = EvalResult.model_validate(itm)
+        except Exception as e:
+            logger.error(f"Failed to validate item {itm_idx} with EvalResult: {itm}. Error: {e}")
+            continue
+
+        sub = ev.submission
+        date_str = None
+        if sub.submit_time is not None:
+            submit_dt = sub.submit_time
+            if not isinstance(submit_dt, pd.Timestamp):
+                if submit_dt.tzinfo is None:
+                    logger.debug(f"Submission time for {sub.agent_name} is timezone-naive, assuming UTC.")
+                    submit_dt = submit_dt.replace(tzinfo=ZoneInfo("UTC"))
+            date_str = pd.Timestamp(submit_dt).tz_convert(ZoneInfo(timezone)).strftime("%Y-%m-%d")
+        else:
+            date_str = None
+
+        if not ev.results:
+            logger.warning(
+                f"Skipping submission {sub.agent_name} ({sub.username or 'N/A'}) "
+                f"({sub.submit_time or 'N/A'}) due to no results."
+            )
+            continue
+        stats = compute_summary_statistics(
+            suite_config=cfg, split=split, results=ev.results
+        )
+        flat = {}
+        print(f"STATS STATS ASTATAS SD T S T A A {stats}")
+        for key, s_obj in stats.items():
+            parts = key.split("/")
+            if parts[0] == "overall":
+                flat["overall/score"] = _safe_round(getattr(s_obj, 'score', np.nan))
+                flat["overall/cost"] = _safe_round(getattr(s_obj, 'cost', np.nan))
+            elif parts[0] == "tag" and len(parts) > 1:
+                tag_name = parts[1]
+                flat[f"tag/{tag_name}/score"] = _safe_round(getattr(s_obj, 'score', np.nan))
+                flat[f"tag/{tag_name}/cost"] = _safe_round(getattr(s_obj, 'cost', np.nan))
+            elif parts[0] == "task" and len(parts) > 1:
+                task_name = parts[1]
+                score = getattr(s_obj, 'score', np.nan)
+                cost = getattr(s_obj, 'cost', np.nan)
+                score_stderr = getattr(s_obj, 'score_stderr', np.nan)
+                cost_stderr = getattr(s_obj, 'cost_stderr', np.nan)
+
+                flat[f"task/{task_name}/score"] = _safe_round(score)
+                flat[f"task/{task_name}/score_ci"] = _safe_round(score_stderr * 1.96 if pd.notna(score_stderr) else np.nan)
+                flat[f"task/{task_name}/cost"] = _safe_round(cost)
+                flat[f"task/{task_name}/cost_ci"] = _safe_round(cost_stderr * 1.96 if pd.notna(cost_stderr) else np.nan)
+            else:
+                logger.debug(f"Uncommon key structure from compute_summary_statistics: '{key}'. Attempting generic add.")
+                if hasattr(s_obj, 'score'):
+                    flat[f"{key}/score"] = _safe_round(s_obj.score)
+                if hasattr(s_obj, 'cost'):
+                    flat[f"{key}/cost"] = _safe_round(s_obj.cost)
+
+        current_logs_url = None
+        if is_internal and sub.logs_url:
+            current_logs_url = str(sub.logs_url)
+        elif not is_internal and sub.logs_url_public:
+            current_logs_url = str(sub.logs_url_public)
+
+        rows.append(
+            {
+                "agent_name": sub.agent_name or "N/A",
+                "username": sub.username or "N/A",
+                "submit_time": date_str,
+                **flat,
+                "logs_url": current_logs_url,
+            }
+        )
+
+    if not rows:
+        logger.warning(f"No valid rows generated from records_list for split '{split}'. Returning empty DataFrame with placeholder.")
+        expected_pretty_cols = ["Agent", "Submitter", "Date", "Overall Score", "Overall Cost", "Logs"]
+        empty_df = pd.DataFrame({p_col: ["No data"] for p_col in expected_pretty_cols})
+        return empty_df
+
+    df = pd.DataFrame(rows)
+    pretty_cols = {c: _pretty_column_name(c) for c in df.columns if c in df.columns}
+    overview = df.rename(columns=pretty_cols)
+    return overview
+
+def _pretty_column_name(col: str) -> str:
+    """Map raw column name to display name."""
+    # --- Step 1: Fixed, direct mappings ---
+    fixed_mappings = {
+        "submit_time": "Date",
+        "agent_name": "Agent",
+        "username": "Submitter",
+        "logs_url": "Logs",
+        "overall/score": "Overall Score",
+        "overall/cost": "Overall Cost",
+    }
+    if col in fixed_mappings:
+        return fixed_mappings[col]
+
+    # --- Step 2: Define your mapping for informal names to descriptive names ---
+    informal_map = DataTransformer._INFORMAL_TO_FORMAL_NAME_MAP
+
+    # --- Step 3: Dynamic mappings for task or tag columns using the informal_to_formal_name_map ---
+    parts = col.split("/")
+    if len(parts) == 3:
+        item_type, informal_name, metric_suffix = parts #
+
+        formal_name = informal_map.get(informal_name)
+        if formal_name is None:
+            formal_name = informal_name.replace("_", " ").title()
+            print(f"[DEBUG _pretty_column_name] Informal name '{informal_name}' not in map, using fallback: '{formal_name}'")
+
+        if metric_suffix == "score":
+            return f"{formal_name} Score"
+        if metric_suffix == "cost":
+            return f"{formal_name} Cost"
+        if metric_suffix == "score_ci":
+            return f"{formal_name} Score 95% CI"
+        if metric_suffix == "cost_ci":
+            return f"{formal_name} Cost 95% CI"
+
+    # --- Step 4: Fallback for columns that don't match the "type/name/metric" pattern ---
+    if "/" not in col:
+        return col.replace("_", " ").title()
+    else:
+        return parts[-1].replace("_", " ").title()
+
+DEFAULT_Y_COLUMN = "Overall Score"
+DUMMY_X_VALUE_FOR_MISSING_COSTS = 0 # Value to use if x-axis data (costs) is missing
+
+def _plot_scatter_plotly(
+        data: pd.DataFrame,
+        x: Optional[str],
+        y: str,
+        agent_col: str = "Agent"
+) -> go.Figure:
+
+    x_col_to_use = x
+    y_col_to_use = y
+
+    # 1. Check if y-column exists
+    if y_col_to_use not in data.columns:
+        logger.error(
+            f"y-axis column '{y_col_to_use}' MUST exist in DataFrame. "
+            f"Cannot generate plot. Available columns: {data.columns.tolist()}"
+        )
+        return go.Figure()
+
+    # 2. Check if agent_col exists
+    if agent_col not in data.columns:
+        logger.warning(
+            f"Agent column '{agent_col}' not found in DataFrame. "
+            f"Available columns: {data.columns.tolist()}. Returning empty figure."
+        )
+        return go.Figure()
+
+    # 3. Prepare data (make a copy, handle numeric conversion for y)
+    data_plot = data.copy()
+    try:
+        data_plot[y_col_to_use] = pd.to_numeric(data_plot[y_col_to_use], errors='coerce')
+    except Exception as e:
+        logger.error(f"Error converting y-column '{y_col_to_use}' to numeric: {e}. Returning empty figure.")
+        return go.Figure()
+
+    # 4. Handle x-column (costs)
+    x_axis_label = x_col_to_use if x_col_to_use else "Cost (Data N/A)" # Label for the x-axis
+    x_data_is_valid = False
+
+    if x_col_to_use and x_col_to_use in data_plot.columns:
+        try:
+            data_plot[x_col_to_use] = pd.to_numeric(data_plot[x_col_to_use], errors='coerce')
+            # Check if there's any non-NaN data after coercion for x
+            if data_plot[x_col_to_use].notna().any():
+                x_data_is_valid = True
+            else:
+                logger.info(f"x-axis column '{x_col_to_use}' exists but contains all NaN/None values after numeric conversion.")
+        except Exception as e:
+            logger.warning(f"Error converting x-column '{x_col_to_use}' to numeric: {e}. Will use dummy x-values.")
+            # x_data_is_valid remains False
+    else:
+        if x_col_to_use: # Name was provided but column doesn't exist
+            logger.warning(f"x-axis column '{x_col_to_use}' not found in DataFrame.")
+        else: # x (column name) was None
+            logger.info("x-axis column name was not provided (is None).")
+
+    if not x_data_is_valid:
+        logger.info(f"Using dummy x-value '{DUMMY_X_VALUE_FOR_MISSING_COSTS}' for all data points as x-data is missing or invalid.")
+        # Create a new column with the dummy x-value for all rows
+        # Use a unique name for this dummy column to avoid potential clashes
+        dummy_x_col_name = "__dummy_x_for_plotting__"
+        data_plot[dummy_x_col_name] = DUMMY_X_VALUE_FOR_MISSING_COSTS
+        x_col_to_use = dummy_x_col_name # Update x_col_to_use to point to our dummy data
+        x_axis_label = x if x else "Cost (Data N/A)" # Use original x name for label if provided
+        # or a generic label if x was None.
+        # Could also be f"Cost (Fixed at {DUMMY_X_VALUE_FOR_MISSING_COSTS})"
+
+
+    # 5. Drop rows where y is NaN (x is now guaranteed to have values, either real or dummy)
+    data_plot.dropna(subset=[y_col_to_use], inplace=True)
+
+    fig = go.Figure()
+
+    if data_plot.empty:
+        logger.warning(f"No valid data to plot for y='{y_col_to_use}' (and x='{x_col_to_use}') after cleaning NaNs from y.")
+        # Still return a figure object, but it will be empty. Update layout for clarity.
+        fig.update_layout(
+            title=f"{y_col_to_use} vs. {x_axis_label} (No Data)",
+            xaxis=dict(title=x_axis_label, range=[DUMMY_X_VALUE_FOR_MISSING_COSTS - 1, DUMMY_X_VALUE_FOR_MISSING_COSTS + 1] if not x_data_is_valid else None),
+            yaxis=dict(title=y_col_to_use)
+        )
+        return fig
+
+
+    for agent, group in data_plot.groupby(agent_col):
+        hover_x_display = "%{x:.2f}" if x_data_is_valid else str(DUMMY_X_VALUE_FOR_MISSING_COSTS) + " (fixed)"
+        fig.add_trace(go.Scatter(
+            x=group[x_col_to_use],
+            y=group[y_col_to_use],
+            mode='markers',
+            name=str(agent),
+            hovertemplate=f"{x_axis_label}: {hover_x_display}<br>{y_col_to_use}: %{{y:.2f}}<extra>{str(agent)}</extra>",
+            marker=dict(size=10)
+        ))
+
+    # Configure layout
+    xaxis_config = dict(title=x_axis_label)
+    if not x_data_is_valid: # If using dummy x, set a tighter, fixed range for x-axis
+        xaxis_config['range'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS - 1, DUMMY_X_VALUE_FOR_MISSING_COSTS + 1]
+        xaxis_config['tickvals'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS] # Show only one tick at the dummy value
+        xaxis_config['ticktext'] = [str(DUMMY_X_VALUE_FOR_MISSING_COSTS)]
+    else: # Real x-data
+        xaxis_config['rangemode'] = "tozero"
+
+
+    fig.update_layout(
+        title=f"{y_col_to_use} vs. {x_axis_label}",
+        xaxis=xaxis_config,
+        yaxis=dict(title=y_col_to_use, rangemode="tozero"),
+        legend_title_text=agent_col
+    )
+
+    return fig

leaderboard_transformer.py

@@ -0,0 +1,436 @@
+import plotly.graph_objects as go
+import numpy as np
+import pandas as pd
+import logging
+from typing import Optional, Any, Dict, List # Added List
+from zoneinfo import ZoneInfo # Assuming this might be used by SuiteConfig/EvalResult or _get_dataframe
+import json
+import os
+
+logger = logging.getLogger(__name__)
+
+INFORMAL_TO_FORMAL_NAME_MAP = {
+    # Short Names
+    "lit": "Literature Understanding",
+    "data": "Data Analysis",
+    "code": "Code Execution",
+    "discovery": "Discovery",
+
+    # Long Raw Names
+    "arxivdigestables_validation": "Arxivdigestables Validation",
+    "sqa_dev": "Sqa Dev",
+    "litqa2_validation": "Litqa2 Validation",
+    "paper_finder_validation": "Paper Finder Validation",
+    "discoverybench_validation": "Discoverybench Validation",
+    "core_bench_validation": "Core Bench Validation",
+    "ds1000_validation": "DS1000 Validation",
+    "e2e_discovery_validation": "E2E Discovery Validation",
+    "super_validation": "Super Validation",
+}
+
+
+### 2. The Updated Helper Functions ###
+
+def _safe_round(value, digits=2):
+    """Rounds a number if it's a valid float/int, otherwise returns it as is."""
+    return round(value, digits) if isinstance(value, (float, int)) and pd.notna(value) else value
+
+
+def _pretty_column_name(raw_col: str) -> str:
+    """
+    Takes a raw column name from the DataFrame and returns a "pretty" version.
+    Handles three cases:
+    1. Fixed names (e.g., 'User/organization' -> 'Submitter').
+    2. Dynamic names (e.g., 'ds1000_validation score' -> 'DS1000 Validation Score').
+    3. Fallback for any other names.
+    """
+    # Case 1: Handle fixed, special-case mappings first.
+    fixed_mappings = {
+        'Agent': 'Agent',
+        'Agent description': 'Agent Description',
+        'User/organization': 'Submitter',
+        'Submission date': 'Date',
+        'Overall': 'Overall Score',
+        'Overall cost': 'Overall Cost',
+        'Logs': 'Logs'
+    }
+    if raw_col in fixed_mappings:
+        return fixed_mappings[raw_col]
+
+    # Case 2: Handle dynamic names by finding the longest matching base name.
+    # We sort by length (desc) to match 'core_bench_validation' before 'core_bench'.
+    sorted_base_names = sorted(INFORMAL_TO_FORMAL_NAME_MAP.keys(), key=len, reverse=True)
+
+    for base_name in sorted_base_names:
+        if raw_col.startswith(base_name):
+            formal_name = INFORMAL_TO_FORMAL_NAME_MAP[base_name]
+
+            # Get the metric part (e.g., ' score' or ' cost 95% CI')
+            metric_part = raw_col[len(base_name):].strip()
+
+            # Capitalize the metric part correctly (e.g., 'score' -> 'Score')
+            pretty_metric = metric_part.capitalize()
+
+            return f"{formal_name} {pretty_metric}"
+
+    # Case 3: If no specific rule applies, just make it title case.
+    return raw_col.title()
+
+
+def create_pretty_tag_map(raw_tag_map: dict, name_map: dict) -> dict:
+    """
+    Converts a tag map with raw names into a tag map with pretty, formal names.
+
+    Args:
+        raw_tag_map: The map with raw keys and values (e.g., {'lit': ['litqa2_validation']}).
+        name_map: The INFORMAL_TO_FORMAL_NAME_MAP used for translation.
+
+    Returns:
+        A new dictionary with pretty names (e.g., {'Literature Understanding': ['Litqa2 Validation']}).
+    """
+    pretty_map = {}
+    # A reverse map to find raw keys from formal names if needed, though not used here
+    # This is just for understanding; the main logic uses the forward map.
+
+    # Helper to get pretty name with a fallback
+    def get_pretty(raw_name):
+        return name_map.get(raw_name, raw_name.replace("_", " ").title())
+
+    for raw_key, raw_value_list in raw_tag_map.items():
+        pretty_key = get_pretty(raw_key)
+        pretty_value_list = [get_pretty(raw_val) for raw_val in raw_value_list]
+        pretty_map[pretty_key] = sorted(list(set(pretty_value_list)))
+
+    return pretty_map
+
+
+def transform_raw_dataframe(raw_df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Transforms a raw leaderboard DataFrame into a presentation-ready format.
+
+    This function performs two main actions:
+    1. Rounds all numeric metric values (columns containing 'score' or 'cost').
+    2. Renames all columns to a "pretty", human-readable format.
+    Args:
+        raw_df (pd.DataFrame): The DataFrame with raw data and column names
+                               like 'agent_name', 'overall/score', 'tag/code/cost'.
+    Returns:
+        pd.DataFrame: A new DataFrame ready for display.
+    """
+    if not isinstance(raw_df, pd.DataFrame):
+        raise TypeError("Input 'raw_df' must be a pandas DataFrame.")
+
+    df = raw_df.copy()
+
+    # Create the mapping for pretty column names
+    pretty_cols_map = {col: _pretty_column_name(col) for col in df.columns}
+
+    # Rename the columns and return the new DataFrame
+    transformed_df = df.rename(columns=pretty_cols_map)
+    # Apply safe rounding to all metric columns
+    for col in transformed_df.columns:
+        if 'Score' in col or 'Cost' in col:
+            transformed_df[col] = transformed_df[col].apply(_safe_round)
+
+    logger.info("Raw DataFrame transformed: numbers rounded and columns renamed.")
+    return transformed_df
+
+
+class DataTransformer:
+    """
+    Visualizes a pre-processed leaderboard DataFrame.
+
+    This class takes a "pretty" DataFrame and a tag map, and provides
+    methods to view filtered versions of the data and generate plots.
+    """
+    def __init__(self, dataframe: pd.DataFrame, tag_map: dict[str, list[str]]):
+        """
+        Initializes the viewer.
+
+        Args:
+            dataframe (pd.DataFrame): The presentation-ready leaderboard data.
+            tag_map (dict): A map of formal tag names to formal task names.
+        """
+        if not isinstance(dataframe, pd.DataFrame):
+            raise TypeError("Input 'dataframe' must be a pandas DataFrame.")
+        if not isinstance(tag_map, dict):
+            raise TypeError("Input 'tag_map' must be a dictionary.")
+
+        self.data = dataframe
+        self.tag_map = tag_map
+        logger.info(f"DataTransformer initialized with a DataFrame of shape {self.data.shape}.")
+
+
+    def view(
+            self,
+            tag: Optional[str] = "Overall", # Default to "Overall" for clarity
+            use_plotly: bool = False,
+    ) -> tuple[pd.DataFrame, dict[str, go.Figure]]:
+        """
+        Generates a filtered view of the DataFrame and a corresponding scatter plot.
+        """
+        if self.data.empty:
+            logger.warning("No data available to view.")
+            return self.data, {}
+
+        # --- 1. Determine Primary and Group Metrics Based on the Tag ---
+        if tag is None or tag == "Overall":
+            primary_metric = "Overall"
+            group_metrics = list(self.tag_map.keys())
+        else:
+            primary_metric = tag
+            # For a specific tag, the group is its list of sub-tasks.
+            group_metrics = self.tag_map.get(tag, [])
+
+        # --- 2. Sort the DataFrame by the Primary Score ---
+        primary_score_col = f"{primary_metric} Score"
+        df_sorted = self.data
+        if primary_score_col in self.data.columns:
+            df_sorted = self.data.sort_values(primary_score_col, ascending=False, na_position='last')
+
+        # --- 3. Build the List of Columns to Display ---
+        base_cols = ["Agent", "Submitter"]
+        new_cols = ["Openness", "Degree of Control"]
+        ending_cols = ["Date", "Logs"]
+
+        # Start with the primary metric score and cost
+        metrics_to_display = [primary_score_col, f"{primary_metric} Cost"]
+
+        # Add the score and cost for each item in our group
+        for item in group_metrics:
+            metrics_to_display.append(f"{item} Score")
+            metrics_to_display.append(f"{item} Cost")
+
+        # Combine base columns with metric columns, ensuring uniqueness and order
+        final_cols_ordered = base_cols + list(dict.fromkeys(metrics_to_display))+ new_cols + ending_cols
+
+        # Filter to only include columns that actually exist in our DataFrame
+        df_view = df_sorted.copy()
+        for col in final_cols_ordered:
+            if col not in df_view.columns:
+                df_view[col] = pd.NA
+
+        df_view = df_view[final_cols_ordered].reset_index(drop=True)
+
+        # Calculated and add "Categories Attempted" column
+        if primary_metric == "Overall":
+            def calculate_attempted(row):
+                main_categories = ['Literature Understanding', 'Data Analysis', 'Code Execution', 'Discovery']
+                count = sum(1 for category in main_categories if pd.notna(row.get(f"{category} Cost")))
+
+                # Return the formatted string with the correct emoji
+                if count == 4:
+                    return f"4/4 ✅"
+                if count == 0:
+                    return f"0/4 🚫"
+                return f"{count}/4 ⚠️"
+
+            # Apply the function row-wise to create the new column
+            attempted_column = df_view.apply(calculate_attempted, axis=1)
+            # Insert the new column at a nice position (e.g., after "Date")
+            df_view.insert(2, "Categories Attempted", attempted_column)
+        else:
+            total_benchmarks = len(group_metrics)
+            def calculate_benchmarks_attempted(row):
+                # Count how many benchmarks in this category have COST data reported
+                count = sum(1 for benchmark in group_metrics if pd.notna(row.get(f"{benchmark} Cost")))
+                if count == total_benchmarks:
+                    return f"{count}/{total_benchmarks} ✅"
+                elif count == 0:
+                    return f"{count}/{total_benchmarks} 🚫"
+                else:
+                    return f"{count}/{total_benchmarks}⚠️"
+            # Insert the new column, for example, after "Date"
+            df_view.insert(2, "Benchmarks Attempted", df_view.apply(calculate_benchmarks_attempted, axis=1))
+
+
+        # --- 4. Generate the Scatter Plot for the Primary Metric ---
+        plots: dict[str, go.Figure] = {}
+        if use_plotly:
+            primary_cost_col = f"{primary_metric} Cost"
+            # Check if the primary score and cost columns exist in the FINAL view
+            if primary_score_col in df_view.columns and primary_cost_col in df_view.columns:
+                fig = _plot_scatter_plotly(
+                    data=df_view,
+                    x=primary_cost_col,
+                    y=primary_score_col,
+                    agent_col="Agent"
+                )
+                # Use a consistent key for easy retrieval later
+                plots['scatter_plot'] = fig
+            else:
+                logger.warning(
+                    f"Skipping plot for '{primary_metric}': score column '{primary_score_col}' "
+                    f"or cost column '{primary_cost_col}' not found."
+                )
+                # Add an empty figure to avoid downstream errors
+                plots['scatter_plot'] = go.Figure()
+        return df_view, plots
+
+DEFAULT_Y_COLUMN = "Overall Score"
+DUMMY_X_VALUE_FOR_MISSING_COSTS = 0
+
+def _plot_scatter_plotly(
+        data: pd.DataFrame,
+        x: Optional[str],
+        y: str,
+        agent_col: str = "Agent"
+) -> go.Figure:
+
+    # --- Steps 1-4: Data Validation and Preparation ---
+    x_col_to_use = x
+    y_col_to_use = y
+
+    if y_col_to_use not in data.columns:
+        logger.error(f"y-axis column '{y_col_to_use}' not found.")
+        return go.Figure()
+    if agent_col not in data.columns:
+        logger.warning(f"Agent column '{agent_col}' not found.")
+        return go.Figure()
+
+    data_plot = data.copy()
+    data_plot[y_col_to_use] = pd.to_numeric(data_plot[y_col_to_use], errors='coerce')
+
+    x_axis_label = x if x else "Cost (Data N/A)"
+    x_data_is_valid = False
+    if x and x in data_plot.columns:
+        try:
+            data_plot[x_col_to_use] = pd.to_numeric(data_plot[x_col_to_use], errors='coerce')
+            if data_plot[x_col_to_use].notna().any():
+                x_data_is_valid = True
+        except Exception as e:
+            logger.warning(f"Error converting x-column '{x_col_to_use}' to numeric: {e}")
+
+    if not x_data_is_valid:
+        dummy_x_col_name = "__dummy_x_for_plotting__"
+        data_plot[dummy_x_col_name] = DUMMY_X_VALUE_FOR_MISSING_COSTS
+        x_col_to_use = dummy_x_col_name
+        logger.info("Using dummy x-values for plotting.")
+
+    # --- Step 5: Clean Data and Initialize Figure ---
+    data_plot.dropna(subset=[y_col_to_use, x_col_to_use], inplace=True)
+    fig = go.Figure()
+    if data_plot.empty:
+        logger.warning(f"No valid data to plot for y='{y_col_to_use}' and x='{x_col_to_use}'.")
+        return fig
+
+    # Step 6 - Calculate and Draw the Efficiency Frontier Line ---
+    if x_data_is_valid:
+        # Sort by cost (ascending), then by score (descending) to break ties
+        sorted_data = data_plot.sort_values(by=[x_col_to_use, y_col_to_use], ascending=[True, False])
+
+        frontier_points = []
+        max_score_so_far = float('-inf')
+
+        for index, row in sorted_data.iterrows():
+            score = row[y_col_to_use]
+            # If this point offers a better score than any we've seen before,
+            # it's part of the frontier.
+            if score > max_score_so_far:
+                frontier_points.append({'x': row[x_col_to_use], 'y': score})
+                max_score_so_far = score
+
+        # Add the frontier line trace to the plot if we found any points
+        if frontier_points:
+            frontier_df = pd.DataFrame(frontier_points)
+            fig.add_trace(go.Scatter(
+                x=frontier_df['x'],
+                y=frontier_df['y'],
+                mode='lines',
+                name='Efficiency Frontier',
+                line=dict(color='firebrick', width=2, dash='dash'),
+                hoverinfo='skip' # The line doesn't need a hover tooltip
+            ))
+
+    # --- Step 7: Plot Individual Agent Markers (No changes here) ---
+    for agent, group in data_plot.groupby(agent_col):
+        hover_x_display = "%{x:.2f}" if x_data_is_valid else "N/A"
+        fig.add_trace(go.Scatter(
+            x=group[x_col_to_use],
+            y=group[y_col_to_use],
+            mode='markers',
+            name=str(agent),
+            hovertemplate=f"<b>{str(agent)}</b><br>{x_axis_label}: {hover_x_display}<br>{y_col_to_use}: %{{y:.2f}}""<extra></extra>",
+            marker=dict(size=10, opacity=0.8)
+        ))
+
+    # --- Step 8: Configure Layout (No changes here) ---
+    xaxis_config = dict(title=x_axis_label)
+    if not x_data_is_valid:
+        xaxis_config['range'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS - 1, DUMMY_X_VALUE_FOR_MISSING_COSTS + 1]
+        xaxis_config['tickvals'] = [DUMMY_X_VALUE_FOR_MISSING_COSTS]
+    else:
+        xaxis_config['rangemode'] = "tozero"
+
+    fig.update_layout(
+        title=f"{y_col_to_use} vs. {x_axis_label}",
+        xaxis=xaxis_config,
+        yaxis=dict(title=y_col_to_use, rangemode="tozero"),
+        legend_title_text=agent_col
+    )
+
+    return fig
+
+def format_cost_column(df: pd.DataFrame, cost_col_name: str) -> pd.DataFrame:
+    """
+    Applies custom formatting to a cost column based on its corresponding score column.
+    - If cost is not null, it remains unchanged.
+    - If cost is null but score is not, it becomes "Missing Cost".
+    - If both cost and score are null, it becomes "Not Attempted".
+    Args:
+        df: The DataFrame to modify.
+        cost_col_name: The name of the cost column to format (e.g., "Overall Cost").
+    Returns:
+        The DataFrame with the formatted cost column.
+    """
+    # Find the corresponding score column by replacing "Cost" with "Score"
+    score_col_name = cost_col_name.replace("Cost", "Score")
+
+    # Ensure the score column actually exists to avoid errors
+    if score_col_name not in df.columns:
+        return df # Return the DataFrame unmodified if there's no matching score
+
+    def apply_formatting_logic(row):
+        cost_value = row[cost_col_name]
+        score_value = row[score_col_name]
+        status_color = "#ec4899"
+
+        if pd.notna(cost_value) and isinstance(cost_value, (int, float)):
+            return f"${cost_value:.2f}"
+        elif pd.notna(score_value):
+            return f'<span style="color: {status_color};">Missing Cost</span>'  # Score exists, but cost is missing
+        else:
+            return f'<span style="color: {status_color};">Not Attempted</span>'  # Neither score nor cost exists
+
+    # Apply the logic to the specified cost column and update the DataFrame
+    df[cost_col_name] = df.apply(apply_formatting_logic, axis=1)
+
+    return df
+
+def format_score_column(df: pd.DataFrame, score_col_name: str) -> pd.DataFrame:
+    """
+    Applies custom formatting to a score column for display.
+    - If a score is 0 or NaN, it's displayed as a colored "0".
+    - Other scores are formatted to two decimal places.
+    """
+    status_color = "#ec4899"  # The same color as your other status text
+
+    # First, fill any NaN values with 0 so we only have one case to handle.
+    # We must use reassignment to avoid the SettingWithCopyWarning.
+    df[score_col_name] = df[score_col_name].fillna(0)
+
+    def apply_formatting(score_value):
+        # Now, we just check if the value is 0.
+        if score_value == 0:
+            return f'<span style="color: {status_color};">0.0</span>'
+
+        # For all other numbers, format them for consistency.
+        if isinstance(score_value, (int, float)):
+            return f"{score_value:.2f}"
+
+        # Fallback for any unexpected non-numeric data
+        return score_value
+
+    # Apply the formatting and return the updated DataFrame
+    return df.assign(**{score_col_name: df[score_col_name].apply(apply_formatting)})
+

leaderboard_viewer.py

@@ -0,0 +1,319 @@
+"""
+View and plot leaderboard results.
+"""
+
+import logging
+from typing import Optional
+from zoneinfo import ZoneInfo
+
+import datasets
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import seaborn as sns
+
+from agenteval import compute_summary_statistics
+from agenteval.config import SuiteConfig
+from agenteval.models import EvalResult
+
+logger = logging.getLogger(__name__)
+
+
+class LeaderboardViewer:
+    """
+    Load and visualize leaderboard for a given HF dataset split.
+    """
+
+    def __init__(
+            self, repo_id: str, config: str, split: str, is_internal: bool = False
+    ):
+        self._repo_id = repo_id
+        self._config = config
+        self._split = split
+        self._internal = is_internal
+
+        # build suite_config and mapping from tags to tasks from the first result
+        # TODO: Verify the sort order
+        ds = datasets.load_dataset(repo_id, name=config).get(split)
+        if not ds:
+            raise ValueError(f"Split '{split}' not found in dataset results")
+        suite = EvalResult.model_validate(ds[0]).suite_config
+        self._cfg = suite
+        self.tag_map: dict[str, list[str]] = {}
+        for task in suite.get_tasks(split):
+            for t in task.tags or []:
+                self.tag_map.setdefault(t, []).append(task.name)
+
+    def _load(self):
+        results = datasets.load_dataset(self._repo_id, name=self._config)
+        overview = _get_dataframe(
+            eval_results=results,
+            split=self._split,
+            is_internal=self._internal,
+            suite_config=self._cfg,
+        )
+        return overview, self.tag_map
+
+    def view(
+            self, tag: Optional[str] = None, with_plots: bool = False
+    ) -> tuple[pd.DataFrame, dict[str, plt.Figure]]:
+        """
+        If tag is None, primary="Overall" and group=all tags.
+        Otherwise primary=tag and group=tasks under that tag.
+        """
+        data, tag_map = self._load()
+        cols = [
+            "Agent",
+            "Submitter",
+            "Completeness",
+            "LLM Base",
+            "Openness" ,
+            "Date",
+            "Logs",
+        ]
+
+        # choose primary metric and its sub‐group
+        if tag is None:
+            primary = "Overall"
+            group = list(tag_map.keys())
+        else:
+            primary = tag
+            group = tag_map.get(tag, [])
+        data = data.sort_values(primary, ascending=False)
+
+        # build full metric list: primary + its cost + each member and its cost
+        metrics = [primary, f"{primary} cost"] + [
+            m for t in group for m in (t, f"{t} cost")
+        ]
+
+        # filter to relevant columns
+        ci_cols = [f"{m} 95% CI" for m in metrics if f"{m} 95% CI" in data.columns]
+        df = data.loc[
+             :,
+             cols + [c for c in metrics if c in data.columns] + ci_cols,
+             ].reset_index(drop=True)
+
+        plots: dict[str, plt.Figure] = {}
+        if with_plots:
+            avail = [c for c in metrics if c in df.columns]
+            for m in [primary] + group:
+                x, y = f"{m} cost", m
+                if x in df.columns and y in df.columns:
+                    plots[f"scatter_{m}"] = _plot_scatter(
+                        df, x=x, y=y, agent_col="Agent"
+                    )
+
+        return df, plots
+
+
+def _get_dataframe(
+        eval_results: datasets.DatasetDict,
+        split: str,
+        is_internal: bool,
+        suite_config: SuiteConfig,
+        timezone: str = "US/Pacific",
+) -> pd.DataFrame:
+    """
+    Load leaderboard results from the given dataset split and return a DataFrame.
+    """
+    ds = eval_results.get(split)
+    if not ds:
+        cols = ["agent_name", "agent_description", "username", "submit_time"]
+        pretty = [_pretty_column_name(c) for c in cols]
+        empty = pd.DataFrame({c: ["No data"] for c in pretty})
+        return empty
+
+    cfg = suite_config
+
+    rows = []
+    for itm in ds:
+        ev = EvalResult.model_validate(itm)
+        sub = ev.submission
+        # only format if submit_time present, else leave as None
+        ts = sub.submit_time
+        if ts is not None:
+            date = ts.astimezone(ZoneInfo(timezone)).strftime("%Y-%m-%d")
+        else:
+            date = None
+
+        if not ev.results:
+            logger.warning(
+                f"Skipping submission {sub.agent_name} ({sub.username}) "
+                f"({sub.submit_time}) with no results"
+            )
+            continue
+        stats = compute_summary_statistics(
+            suite_config=cfg, split=split, results=ev.results
+        )
+        flat = {}
+        for key, s in stats.items():
+            parts = key.split("/")
+            if parts[0] == "overall":
+                flat["overall/score"], flat["overall/cost"] = s.score, s.cost
+            elif parts[0] == "tag":
+                flat[f"tag/{parts[1]}/score"], flat[f"tag/{parts[1]}/cost"] = (
+                    s.score,
+                    s.cost,
+                )
+            else:  # task
+                t0 = parts[1]
+                # compute 95% CI half-width from stderr
+                flat.update(
+                    {
+                        f"task/{t0}/score": s.score,
+                        f"task/{t0}/score_ci": (
+                            (s.score_stderr * 1.96)
+                            if s.score_stderr is not None
+                            else np.nan
+                        ),
+                        f"task/{t0}/cost": s.cost,
+                        f"task/{t0}/cost_ci": (
+                            (s.cost_stderr * 1.96)
+                            if s.cost_stderr is not None
+                            else np.nan
+                        ),
+                    }
+                )
+
+        rows.append(
+            {
+                "agent_name": sub.agent_name,
+                "username": sub.username or "",
+                "submit_time": date,
+                **flat,
+                "logs_url": sub.logs_url if is_internal else sub.logs_url_public,
+            }
+        )
+
+    df = pd.DataFrame(rows)
+
+    # prepare pretty column mapping
+    pretty_cols = {c: _pretty_column_name(c) for c in df.columns}
+
+    # construct overview table with human-friendly names
+    overview = df.rename(columns=pretty_cols)
+
+    return overview
+
+
+def _pretty_column_name(col: str) -> str:
+    """Map raw column name to display name."""
+    # fixed mappings
+    mapping = {
+        "submit_time": "Date",
+        "agent_name": "Agent",
+        "username": "User/organization",
+        "logs_url": "Logs",
+        "overall/score": "Score",
+        "overall/cost": "Cost (USD)",
+    }
+    if col in mapping:
+        return mapping[col]
+    # dynamic: task/{name}/{metric} or tag/{name}/{metric}
+    parts = col.split("/")
+    if len(parts) == 3:
+        _, name, metric = parts
+        if metric == "score":
+            return name
+        if metric == "cost":
+            return f"{name} cost"
+        if metric == "score_ci":
+            return f"{name} 95% CI"
+        if metric == "cost_ci":
+            return f"{name} cost 95% CI"
+    # fallback to last segment
+    return parts[-1]
+
+
+
+def _plot_scatter(
+        data: pd.DataFrame,
+        x: str,  # Cost column name (e.g., "Overall cost")
+        y: str,  # Score column name (e.g., "Overall score")
+        agent_col: str,
+) -> plt.Figure:
+    """Scatter plot of agent results, showing score vs cost with Pareto frontier."""
+    fig, ax = plt.subplots(figsize=(20,7))
+
+    # Make a copy for manipulation to find frontier without affecting original data
+    plot_data = data.copy()
+
+    # Ensure score (y) and cost (x) are numeric and drop NaNs for frontier calculation
+    plot_data[y] = pd.to_numeric(plot_data[y], errors='coerce')
+    plot_data[x] = pd.to_numeric(plot_data[x], errors='coerce')
+    frontier_data = plot_data.dropna(subset=[y, x])
+
+    if not frontier_data.empty:
+        # Sort by cost (x) ascending, then by score (y) descending for tie-breaking
+        frontier_data = frontier_data.sort_values(by=[x, y], ascending=[True, False])
+
+        pareto_points = []
+        max_score_at_cost = -np.inf  # Initialize with negative infinity
+
+        for index, row in frontier_data.iterrows():
+            current_score = row[y]
+            current_cost = row[x]
+            # Only add point if it offers a higher score than any previous point
+            # on the frontier with less or equal cost (implicit by sorting).
+            # More strictly, for a point to be on the frontier here, it must improve the score.
+            if current_score > max_score_at_cost:
+                # Optional: If allowing same score but lower cost (already handled by sort somewhat)
+                # you might need to check if a point with same score but lower cost exists
+                # For this algorithm, we simply take points that strictly increase score.
+                pareto_points.append(row)
+                max_score_at_cost = current_score
+
+        if pareto_points:
+            pareto_df = pd.DataFrame(pareto_points)
+            # Sort pareto_df by cost again just to be sure for plotting line
+            pareto_df = pareto_df.sort_values(by=x)
+            # Plot the Pareto frontier line
+            ax.plot(pareto_df[x], pareto_df[y], marker='o', linestyle='-', color='red', alpha=0.7, linewidth=2, markersize=5, label='Pareto Frontier')
+
+    # Plot all data points
+    sns.scatterplot(data=data, x=x, y=y, hue=agent_col, s=100, ax=ax, legend="auto")
+
+    # Error bars (if they exist)
+    x_ci_col = f"{x} 95% CI"
+    y_ci_col = f"{y} 95% CI"
+    if x_ci_col in data.columns or y_ci_col in data.columns:
+        # Filter data for error bars to only include rows present in the original 'data'
+        # This is important if 'frontier_data' subset was used for some logic but error bars are for all.
+        error_bar_data = data.copy() # Use original data for error bars
+        error_bar_data[x_ci_col] = pd.to_numeric(error_bar_data.get(x_ci_col), errors='coerce')
+        error_bar_data[y_ci_col] = pd.to_numeric(error_bar_data.get(y_ci_col), errors='coerce')
+
+        ax.errorbar(
+            x=error_bar_data[x], # Use original data's x
+            y=error_bar_data[y], # Use original data's y
+            xerr=error_bar_data.get(x_ci_col),
+            yerr=error_bar_data.get(y_ci_col),
+            fmt="none",
+            ecolor="gray",
+            alpha=0.5,
+            capsize=3,
+            zorder=0 # Draw error bars behind scatter points
+        )
+
+    ax.set_xlim(left=0)
+    ax.set_ylim(bottom=0) # Scores and costs are typically non-negative
+    ax.set_xlabel(x) # x is cost
+    ax.set_ylabel(y) # y is score
+
+    # Adjust legend: Get handles and labels from seaborn plot, then add frontier's
+    handles, labels = ax.get_legend_handles_labels()
+    # Check if "Pareto Frontier" was actually plotted and add its handle/label if so
+    if pareto_points and "Pareto Frontier" not in labels: # Avoid duplicate legend items
+        # Find the frontier line object to get its handle
+        frontier_line = next((line for line in ax.get_lines() if line.get_label() == 'Pareto Frontier'), None)
+        if frontier_line:
+            handles.append(frontier_line)
+            labels.append('Pareto Frontier')
+
+    ax.legend(handles=handles, labels=labels, title=agent_col, bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0.)
+
+    plt.tight_layout(rect=[0, 0, 0.85, 1])
+    return fig
+
+
+__all__ = ["LeaderboardViewer"]

literature_understanding.py

@@ -0,0 +1,54 @@
+import gradio as gr
+import pandas as pd
+
+# Import our UI factories and the data loader
+from ui_components import create_leaderboard_display, create_benchmark_details_display, get_full_leaderboard_data
+
+# Define the category for this page
+CATEGORY_NAME = "Literature Understanding"
+
+with gr.Blocks() as demo:
+    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+
+    # --- This page now has two main sections: Validation and Test ---
+    with gr.Tabs():
+        with gr.Tab("Results: Validation"):
+            # 1. Load all necessary data for the "validation" split ONCE.
+            validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+
+            if not validation_df.empty:
+                # 2. Render the main category display using the loaded data.
+                create_leaderboard_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="validation"
+                )
+
+                # 3. Render the detailed breakdown for each benchmark in the category.
+                create_benchmark_details_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for validation split.")
+
+        with gr.Tab("Results: Test"):
+            # Repeat the process for the "test" split
+            test_df, test_tag_map = get_full_leaderboard_data("test")
+
+            if not test_df.empty:
+                create_leaderboard_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME,
+                    split_name="test"
+                )
+                create_benchmark_details_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME
+                )
+            else:
+                gr.Markdown("No data available for test split.")

main_page.py

@@ -0,0 +1,375 @@
+import matplotlib
+matplotlib.use('Agg')
+
+import os
+import shutil
+import tarfile
+import tempfile
+from datetime import datetime, timedelta, timezone
+from email.utils import parseaddr
+from pathlib import Path
+# from zoneinfo import ZoneInfo # LeaderboardViewer uses this, ensure it's available
+
+import gradio as gr
+import requests
+from agenteval import (
+    # compute_summary_statistics, # This will now be used by LeaderboardViewer
+    process_eval_logs,
+    upload_folder_to_hf,
+    upload_summary_to_hf,
+)
+from agenteval.models import EvalResult # Used by submission and LeaderboardViewer (implicitly)
+from agenteval.leaderboard.upload import sanitize_path_component
+from datasets import Dataset, DatasetDict, VerificationMode, load_dataset # load_dataset used by LV
+from datasets.data_files import EmptyDatasetError
+from huggingface_hub import HfApi
+
+from ui_components import create_leaderboard_display, get_full_leaderboard_data
+
+from content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    INTRODUCTION_TEXT,
+    SUBMISSION_TEXT,
+    INTRO_PARAGRAPH,
+    SCATTER_DISCLAIMER,
+    format_error,
+    format_log,
+    format_warning,
+)
+
+# --- Constants and Configuration  ---
+LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
+CONFIG_NAME = "1.0.0-dev1" # This corresponds to 'config' in LeaderboardViewer
+IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
+
+OWNER = "allenai"
+PROJECT_NAME = "asta-bench" + ("-internal" if IS_INTERNAL else "")
+SUBMISSION_DATASET = f"{OWNER}/{PROJECT_NAME}-submissions"
+SUBMISSION_DATASET_PUBLIC = f"{OWNER}/{PROJECT_NAME}-submissions-public"
+CONTACT_DATASET = f"{OWNER}/{PROJECT_NAME}-contact-info"
+RESULTS_DATASET = f"{OWNER}/{PROJECT_NAME}-results" # This is the repo_id for LeaderboardViewer
+LEADERBOARD_PATH = f"{OWNER}/{PROJECT_NAME}-leaderboard"
+
+if LOCAL_DEBUG:
+    DATA_DIR = os.path.join(os.path.dirname(__file__), "data", CONFIG_NAME)
+else:
+    DATA_DIR = "/home/user/data/" + CONFIG_NAME
+EXTRACTED_DATA_DIR = os.path.join(DATA_DIR, "extracted")
+
+api = HfApi()
+MAX_UPLOAD_BYTES = 100 * 1024**2
+AGENTEVAL_MANIFEST_NAME = "agenteval.json"
+os.makedirs(EXTRACTED_DATA_DIR, exist_ok=True)
+
+# --- Global State for Viewers (simple caching) ---
+CACHED_VIEWERS = {}
+CACHED_TAG_MAPS = {}
+
+# --- Submission Logic (largely unchanged from original, ensure EvalResult and other deps are fine) ---
+def try_load_dataset_submission(*args, **kwargs) -> DatasetDict: # Renamed to avoid conflict if LV has one
+    try:
+        return load_dataset(*args, **kwargs)
+    except EmptyDatasetError:
+        return DatasetDict()
+    except ValueError: # Handles cases where dataset is empty or ill-formed
+        return DatasetDict()
+
+def checked_upload_folder(
+        api_hf: HfApi, # Renamed to avoid conflict with global api
+        folder_path: str,
+        repo_id: str,
+        config_name_ul: str, # Renamed
+        split_ul: str, # Renamed
+        submission_name_ul: str, # Renamed
+) -> str:
+    total = 0
+    for root, _, files in os.walk(folder_path):
+        for f_ul in files: # Renamed
+            total += os.path.getsize(os.path.join(root, f_ul))
+            if total > MAX_UPLOAD_BYTES:
+                raise ValueError(
+                    f"Upload too large: exceeds {MAX_UPLOAD_BYTES // (1024**2)} MB limit."
+                )
+    return upload_folder_to_hf(
+        api=api_hf, # Use renamed parameter
+        folder_path=folder_path,
+        repo_id=repo_id,
+        config_name=config_name_ul,
+        split=split_ul,
+        submission_name=submission_name_ul,
+    )
+
+def add_new_eval(
+        val_or_test: str,
+        agent_name: str | None,
+        agent_description: str,
+        agent_url: str,
+        openness: str | None,
+        degree_of_control: str | None,
+        path_to_file: tempfile._TemporaryFileWrapper | None,
+        username: str,
+        mail: str,
+        profile: gr.OAuthProfile,
+        # We need global eval_results for checks; this might need rethinking if it's purely display driven now
+        # For now, let's assume we still load it for submission checks
+):
+    # Load current eval_results for submission checks
+    # This is a bit redundant if display part reloads it, but submission needs its own consistent view
+    current_eval_results_for_submission = try_load_dataset_submission(
+        RESULTS_DATASET,
+        CONFIG_NAME,
+        download_mode="force_redownload", # Or a less aggressive mode
+        verification_mode=VerificationMode.NO_CHECKS,
+        trust_remote_code=True,
+    )
+    if not agent_name:
+        return format_warning("Please provide an agent name.")
+
+    submission_time = datetime.now(timezone.utc)
+    if not username or username.strip() == "":
+        username = profile.username # Default to HF username
+
+    # User account age check
+    try:
+        user_data_resp = requests.get(f"https://huggingface.co/api/users/{profile.username}/overview")
+        user_data_resp.raise_for_status()
+        creation_date_str = user_data_resp.json()["createdAt"]
+        created_at = datetime.strptime(creation_date_str, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
+        if submission_time - created_at < timedelta(days=60):
+            return format_error("This account is not authorized to submit here (account too new).")
+    except Exception as e:
+        print(f"Error checking user account age: {e}")
+        return format_error("Could not verify account age. Please try again later.")
+
+    # Submission frequency check
+    contact_infos = try_load_dataset_submission(
+        CONTACT_DATASET, CONFIG_NAME, download_mode="force_redownload",
+        verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True
+    )
+    user_submission_dates = sorted(
+        datetime.fromisoformat(row["submit_time"])
+        for row in contact_infos.get(val_or_test, []) if row["username_auth"] == profile.username
+    )
+    if user_submission_dates and (submission_time - user_submission_dates[-1] < timedelta(days=1)):
+        return format_error("You already submitted once in the last 24h for this split; please try again later.")
+
+    # Email validation
+    _, parsed_mail = parseaddr(mail)
+    if "@" not in parsed_mail:
+        return format_warning("Please provide a valid email address.")
+
+    # Duplicate submission check
+    if val_or_test in current_eval_results_for_submission and len(current_eval_results_for_submission[val_or_test]) > 0:
+        existing_submissions = current_eval_results_for_submission[val_or_test].to_dict().get("submission", [])
+        for sub_item in existing_submissions:
+            if (sub_item.get("agent_name", "").lower() == agent_name.lower() and
+                    sub_item.get("username", "").lower() == username.lower()):
+                return format_warning("This agent name by this user has already been submitted to this split.")
+
+    if path_to_file is None:
+        return format_warning("Please attach a .tar.gz file.")
+
+    safe_username = sanitize_path_component(username)
+    safe_agent_name = sanitize_path_component(agent_name)
+    extracted_dir = os.path.join(EXTRACTED_DATA_DIR, f"{safe_username}_{safe_agent_name}")
+
+    # File extraction
+    if not LOCAL_DEBUG:
+        try:
+            if os.path.exists(extracted_dir): shutil.rmtree(extracted_dir)
+            os.makedirs(extracted_dir, exist_ok=True)
+            with tarfile.open(path_to_file.name, "r:gz") as tar:
+                members_extracted = 0
+                for member in tar.getmembers():
+                    if not member.isreg(): continue
+                    fname = os.path.basename(member.name)
+                    if not fname or fname.startswith("."): continue
+                    fobj = tar.extractfile(member)
+                    if not fobj: continue
+                    with open(os.path.join(extracted_dir, fname), "wb") as out:
+                        out.write(fobj.read())
+                    members_extracted +=1
+                if members_extracted == 0:
+                    return format_error("Submission tarball is empty or contains no valid files.")
+        except Exception as e:
+            return format_error(f"Error extracting file: {e}. Ensure it's a valid .tar.gz.")
+    else: print("mock extracted file", flush=True)
+
+
+    submission_name = f"{safe_username}_{safe_agent_name}_{submission_time.strftime('%Y-%m-%d_%H-%M-%S')}"
+
+    # 1. Upload raw (unscored) submission files
+    if not LOCAL_DEBUG:
+        try:
+            checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET, CONFIG_NAME, val_or_test, submission_name)
+        except ValueError as e: return format_error(str(e))
+        except Exception as e: return format_error(f"Failed to upload raw submission: {e}")
+    else: print("mock uploaded raw submission", flush=True)
+
+    # 2. Save contact information
+    contact_info = {
+        "agent_name": agent_name, "agent_description": agent_description, "url": agent_url,
+        "username": username, "username_auth": profile.username, "mail": mail,
+        "submit_time": submission_time.isoformat(),
+    }
+    if val_or_test in contact_infos:
+        contact_infos[val_or_test] = contact_infos[val_or_test].add_item(contact_info)
+    else:
+        contact_infos[val_or_test] = Dataset.from_list([contact_info])
+
+    if not LOCAL_DEBUG:
+        try:
+            contact_infos.push_to_hub(CONTACT_DATASET, config_name=CONFIG_NAME)
+        except Exception as e: return format_warning(f"Submission recorded, but contact info failed to save: {e}")
+    else: print("mock uploaded contact info", flush=True)
+
+
+    # 3. Process and score the submission
+    eval_result_obj = None # Define to avoid NameError
+    try:
+        json_path = Path(extracted_dir) / AGENTEVAL_MANIFEST_NAME
+        if not json_path.exists():
+            return format_error(f"Missing manifest {AGENTEVAL_MANIFEST_NAME} in submission.")
+
+        eval_result_obj = EvalResult.model_validate_json(json_path.read_text(encoding="utf-8"))
+        if eval_result_obj.suite_config.version != CONFIG_NAME:
+            return format_error(f"Suite version mismatch: expected {CONFIG_NAME}, got {eval_result_obj.suite_config.version}.")
+        if eval_result_obj.split != val_or_test:
+            return format_error(f"Split mismatch: expected {val_or_test}, got {eval_result_obj.split}.")
+
+        # Re-compute results from logs for integrity
+        eval_result_obj.results = process_eval_logs(extracted_dir)[0] # Assuming process_eval_logs returns a tuple/list
+        eval_result_obj.save_json(str(json_path)) # Save the re-processed manifest
+
+    except Exception as e:
+        return format_error(f"Error scoring submission: {e}. Check manifest and log files.")
+
+    # 4. Upload scored submission files
+    logs_url_private_val, logs_url_public_val = None, None
+    scored_submission_name = f"{submission_name}_scored"
+    if not LOCAL_DEBUG:
+        try:
+            logs_url_private_val = checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET, CONFIG_NAME, val_or_test, scored_submission_name)
+            if val_or_test == "validation" and not IS_INTERNAL: # Public copy for validation
+                logs_url_public_val = checked_upload_folder(api, extracted_dir, SUBMISSION_DATASET_PUBLIC, CONFIG_NAME, val_or_test, scored_submission_name)
+        except ValueError as e: return format_error(str(e))
+        except Exception as e: return format_error(f"Failed to upload scored submission: {e}")
+    else: print("mock uploaded scored submission", flush=True)
+
+
+    # Update EvalResult with submission details
+    eval_result_obj.submission.agent_name = agent_name
+    eval_result_obj.submission.agent_description = agent_description
+    eval_result_obj.submission.agent_url = agent_url
+    eval_result_obj.submission.openness = openness
+    eval_result_obj.submission.degree_of_control = degree_of_control
+    eval_result_obj.submission.username = username
+    eval_result_obj.submission.submit_time = submission_time
+    eval_result_obj.submission.logs_url = logs_url_private_val
+    eval_result_obj.submission.logs_url_public = logs_url_public_val
+
+    # 5. Upload summary statistics to RESULTS_DATASET (for the leaderboard)
+    if not LOCAL_DEBUG:
+        try:
+            upload_summary_to_hf(api, eval_result_obj, RESULTS_DATASET, CONFIG_NAME, val_or_test, scored_submission_name)
+        except Exception as e:
+            return format_error(f"Failed to upload summary results to leaderboard: {e}")
+    else: print("mock uploaded results to lb", flush=True)
+
+    # Invalidate viewer cache for the split that was updated
+    if val_or_test in CACHED_VIEWERS:
+        del CACHED_VIEWERS[val_or_test]
+    if val_or_test in CACHED_TAG_MAPS:
+        del CACHED_TAG_MAPS[val_or_test]
+
+
+    return format_log(
+        f"Agent '{agent_name}' submitted successfully by '{username}' to '{val_or_test}' split. "
+        "Please refresh the leaderboard in a few moments. It may take some time for changes to propagate."
+    )
+
+with gr.Blocks() as demo:
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    gr.HTML(INTRO_PARAGRAPH, elem_id="intro-paragraph")
+
+    # --- Submission Accordion ---
+    with gr.Accordion("🚀 Submit a new agent for evaluation", open=False, elem_classes="submission-accordion"):
+        gr.Markdown(SUBMISSION_TEXT, elem_id="markdown-text")
+        with gr.Row():
+            with gr.Column():
+                level_of_test_radio = gr.Radio(["validation", "test"], value="validation", label="Split")
+                agent_name_tb = gr.Textbox(label="Agent Name")
+                agent_desc_tb = gr.Textbox(label="Agent Description")
+                agent_url_tb = gr.Textbox(label="URL to Agent Information")
+                openness_radio = gr.Radio(["Open Source", "API", "UI"], value=None, label="Openness of Agent")
+                degree_of_control_radio = gr.Radio(["Standard", "Custom"], value=None, label="Degree of Control")
+            with gr.Column():
+                username_tb = gr.Textbox(label="Organization or User Name (Defaults to HF username)")
+                mail_tb = gr.Textbox(label="Contact Email (Private, for submission issues)")
+                file_upload_comp = gr.File(
+                    label="Submission File (.tar.gz ...)", # Shortened for brevity
+                    file_types=[".gz", ".tar.gz"]
+                )
+        with gr.Row():
+            gr.LoginButton()
+            submit_eval_button = gr.Button("Submit Evaluation")
+        submission_result = gr.Markdown()
+
+        submit_eval_button.click(
+            add_new_eval,
+            [
+                level_of_test_radio,
+                agent_name_tb,
+                agent_desc_tb,
+                agent_url_tb,
+                openness_radio,
+                degree_of_control_radio,
+                file_upload_comp,
+                username_tb,
+                mail_tb
+            ],
+            submission_result,
+        )
+
+    # --- Leaderboard Display Section ---
+    gr.Markdown("---")
+    CATEGORY_NAME = "Overall"
+    gr.Markdown(f"## {CATEGORY_NAME} Leaderboard Results")
+
+    with gr.Tabs() as tabs:
+        with gr.Tab("Results: Validation"):
+            # 1. Load all necessary data for the "validation" split ONCE.
+            validation_df, validation_tag_map = get_full_leaderboard_data("validation")
+
+            # Check if data was loaded successfully before trying to display it
+            if not validation_df.empty:
+                # 2. Render the display by calling the factory with the loaded data.
+                create_leaderboard_display(
+                    full_df=validation_df,
+                    tag_map=validation_tag_map,
+                    category_name=CATEGORY_NAME, # Use our constant
+                    split_name="validation"
+                )
+            else:
+                # Display a message if no data is available
+                gr.Markdown("No data available for validation split.")
+
+        with gr.Tab("Results: Test"):
+            test_df, test_tag_map = get_full_leaderboard_data("test")
+            if not test_df.empty:
+                create_leaderboard_display(
+                    full_df=test_df,
+                    tag_map=test_tag_map,
+                    category_name=CATEGORY_NAME, # Use our constant
+                    split_name="test"
+                )
+            else:
+                gr.Markdown("No data available for test split.")
+
+    with gr.Accordion("📙 Citation", open=False):
+        gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button-main", interactive=False)
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -1,5 +1,5 @@
 datasets
-gradio[oauth]
+gradio[oauth]==5.30.0
 huggingface-hub
 APScheduler
-agent-eval @ git+https://github.com/allenai/agent-eval.git@d302cb5d0ba983ae5f0764c53fde4e017118d0df#egg=agent-eval
+agent-eval==0.1.9

ui_components.py

@@ -0,0 +1,293 @@
+import gradio as gr
+from gradio.events import SelectData
+import pandas as pd
+import plotly.graph_objects as go
+import os
+
+from agenteval.leaderboard.view import LeaderboardViewer
+from huggingface_hub import HfApi
+
+from leaderboard_transformer import DataTransformer, transform_raw_dataframe, create_pretty_tag_map, INFORMAL_TO_FORMAL_NAME_MAP, _plot_scatter_plotly, format_cost_column, format_score_column
+from content import (
+    SCATTER_DISCLAIMER,
+    format_error,
+    format_log,
+    format_warning,
+    hf_uri_to_web_url,
+    hyperlink,
+)
+
+# --- Constants and Configuration  ---
+LOCAL_DEBUG = not (os.environ.get("system") == "spaces")
+CONFIG_NAME = "1.0.0-dev1" # This corresponds to 'config' in LeaderboardViewer
+IS_INTERNAL = os.environ.get("IS_INTERNAL", "false").lower() == "true"
+
+OWNER = "allenai"
+PROJECT_NAME = "asta-bench" + ("-internal" if IS_INTERNAL else "")
+SUBMISSION_DATASET = f"{OWNER}/{PROJECT_NAME}-submissions"
+SUBMISSION_DATASET_PUBLIC = f"{OWNER}/{PROJECT_NAME}-submissions-public"
+CONTACT_DATASET = f"{OWNER}/{PROJECT_NAME}-contact-info"
+RESULTS_DATASET = f"{OWNER}/{PROJECT_NAME}-results" # This is the repo_id for LeaderboardViewer
+LEADERBOARD_PATH = f"{OWNER}/{PROJECT_NAME}-leaderboard"
+
+if LOCAL_DEBUG:
+    DATA_DIR = os.path.join(os.path.dirname(__file__), "data", CONFIG_NAME)
+else:
+    DATA_DIR = "/home/user/data/" + CONFIG_NAME
+EXTRACTED_DATA_DIR = os.path.join(DATA_DIR, "extracted")
+
+api = HfApi()
+MAX_UPLOAD_BYTES = 100 * 1024**2
+AGENTEVAL_MANIFEST_NAME = "agenteval.json"
+os.makedirs(EXTRACTED_DATA_DIR, exist_ok=True)
+
+
+# --- Global State for Viewers (simple caching) ---
+CACHED_VIEWERS = {}
+CACHED_TAG_MAPS = {}
+
+# --- New Helper Class to Solve the Type Mismatch Bug ---
+class DummyViewer:
+    """A mock viewer to be cached on error. It has a ._load() method
+       to ensure it behaves like the real LeaderboardViewer."""
+    def __init__(self, error_df):
+        self._error_df = error_df
+
+    def _load(self):
+        # The _load method returns the error DataFrame and an empty tag map
+        return self._error_df, {}
+
+def get_leaderboard_viewer_instance(split: str):
+    """
+    Fetches the LeaderboardViewer for a split, using a cache to avoid
+    re-downloading data. On error, returns a stable DummyViewer object.
+    """
+    global CACHED_VIEWERS, CACHED_TAG_MAPS
+
+    if split in CACHED_VIEWERS:
+        # Cache hit: return the cached viewer and tag map
+        return CACHED_VIEWERS[split], CACHED_TAG_MAPS.get(split, {"Overall": []})
+
+    # --- Cache miss: try to load data from the source ---
+    try:
+        print(f"Using Hugging Face dataset for split '{split}': {RESULTS_DATASET}/{CONFIG_NAME}")
+        viewer = LeaderboardViewer(
+            repo_id=RESULTS_DATASET,
+            config=CONFIG_NAME,
+            split=split,
+            is_internal=IS_INTERNAL
+        )
+
+        # Simplify tag map creation
+        pretty_tag_map = create_pretty_tag_map(viewer.tag_map, INFORMAL_TO_FORMAL_NAME_MAP)
+
+        # Cache the results for next time
+        CACHED_VIEWERS[split] = viewer
+        CACHED_TAG_MAPS[split] = pretty_tag_map # Cache the pretty map directly
+
+        return viewer, pretty_tag_map
+
+    except Exception as e:
+        # On ANY error, create a consistent error message and cache a DummyViewer
+        error_message = f"Error loading data for split '{split}': {e}"
+        print(format_error(error_message))
+
+        dummy_df = pd.DataFrame({"Message": [error_message]})
+        dummy_viewer = DummyViewer(dummy_df)
+        dummy_tag_map = {"Overall": []}
+
+        # Cache the dummy objects so we don't try to fetch again on this run
+        CACHED_VIEWERS[split] = dummy_viewer
+        CACHED_TAG_MAPS[split] = dummy_tag_map
+
+        return dummy_viewer, dummy_tag_map
+
+
+def create_leaderboard_display(
+        full_df: pd.DataFrame,
+        tag_map: dict,
+        category_name: str,
+        split_name: str
+):
+    """
+    This UI factory takes pre-loaded data and renders the main DataFrame and Plot
+    for a given category (e.g., "Overall" or "Literature Understanding").
+    """
+    # 1. Instantiate the transformer and get the specific view for this category.
+    # The function no longer loads data itself; it filters the data it receives.
+    transformer = DataTransformer(full_df, tag_map)
+    df_view, plots_dict = transformer.view(tag=category_name, use_plotly=True)
+    # format cost columns
+    for col in df_view.columns:
+        if "Cost" in col:
+            df_view = format_cost_column(df_view, col)
+
+    # 2. Fill NaN scores with 0
+    for col in df_view.columns:
+        if "Score" in col:
+            df_view = format_score_column(df_view, col)
+    scatter_plot = plots_dict.get('scatter_plot', go.Figure())
+
+    # 2. Define the UI components with the filtered data.
+    df_headers = df_view.columns.tolist()
+    df_datatypes = ["markdown" if col == "Logs" or "Cost" in col or "Score" in col else "str" for col in df_headers]
+
+    dataframe_component = gr.DataFrame(
+        headers=df_headers,
+        value=df_view,
+        datatype=df_datatypes,
+        interactive=False,
+        wrap=True,
+        column_widths=[100, 100, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 70, 75, 75, 50, 50]
+    )
+
+    plot_component = gr.Plot(
+        value=scatter_plot,
+        label=f"Score vs. Cost ({category_name})"
+    )
+    gr.HTML(SCATTER_DISCLAIMER, elem_id="scatter-disclaimer")
+
+    # Return the components so they can be referenced elsewhere.
+    return dataframe_component, plot_component
+
+def get_full_leaderboard_data(split: str) -> tuple[pd.DataFrame, dict]:
+    """
+    Loads and transforms the complete dataset for a given split.
+    This function handles caching and returns the final "pretty" DataFrame and tag map.
+    """
+    # This reuses your existing robust caching logic
+    viewer_or_data, raw_tag_map = get_leaderboard_viewer_instance(split)
+
+    if isinstance(viewer_or_data, (LeaderboardViewer, DummyViewer)):
+        raw_df, _ = viewer_or_data._load()
+        if raw_df.empty:
+            return pd.DataFrame(), {}
+
+        pretty_df = transform_raw_dataframe(raw_df)
+        pretty_tag_map = create_pretty_tag_map(raw_tag_map, INFORMAL_TO_FORMAL_NAME_MAP)
+        if "Logs" in pretty_df.columns:
+            def format_log_entry_to_html(raw_uri):
+                if pd.isna(raw_uri) or raw_uri == "": return ""
+                web_url = hf_uri_to_web_url(str(raw_uri))
+                return hyperlink(web_url, "🔗") if web_url else ""
+
+            # Apply the function to the "Logs" column
+            pretty_df["Logs"] = pretty_df["Logs"].apply(format_log_entry_to_html)
+
+        return pretty_df, pretty_tag_map
+
+    # Fallback for unexpected types
+    return pd.DataFrame(), {}
+
+# --- Detailed Benchmark Display ---
+def create_benchmark_details_display(
+        full_df: pd.DataFrame,
+        tag_map: dict,
+        category_name: str
+):
+    """
+    Generates a detailed breakdown for each benchmark within a given category.
+    For each benchmark, it creates a title, a filtered table, and a scatter plot.
+    Args:
+        full_df (pd.DataFrame): The complete, "pretty" dataframe for the entire split.
+        tag_map (dict): The "pretty" tag map to find the list of benchmarks.
+        category_name (str): The main category to display details for (e.g., "Literature Understanding").
+    """
+    # 1. Get the list of benchmarks for the selected category
+    benchmark_names = tag_map.get(category_name, [])
+
+    if not benchmark_names:
+        gr.Markdown(f"No detailed benchmarks found for the category: {category_name}")
+        return
+
+    gr.Markdown("---")
+    gr.Markdown("## Detailed Benchmark Results")
+
+    # 2. Loop through each benchmark and create its UI components
+    for benchmark_name in benchmark_names:
+        with gr.Blocks():
+            gr.Markdown(f"### {benchmark_name}")
+
+            # 3. Prepare the data for this specific benchmark's table and plot
+            benchmark_score_col = f"{benchmark_name} Score"
+            benchmark_cost_col = f"{benchmark_name} Cost"
+
+            # Define the columns needed for the detailed table
+            table_cols = ['Agent', 'Submitter', 'Date', benchmark_score_col, benchmark_cost_col,'Logs']
+
+            # Filter to only columns that actually exist in the full dataframe
+            existing_table_cols = [col for col in table_cols if col in full_df.columns]
+
+            if benchmark_score_col not in existing_table_cols:
+                gr.Markdown(f"Score data for {benchmark_name} not available.")
+                continue # Skip to the next benchmark if score is missing
+
+            # Create a specific DataFrame for the table view
+            benchmark_table_df = full_df[existing_table_cols].copy()
+            # Calculated and add "Benchmark Attempted" column
+            def check_benchmark_status(row):
+                has_score = pd.notna(row.get(benchmark_score_col))
+                has_cost = pd.notna(row.get(benchmark_cost_col))
+
+                if has_score and has_cost:
+                    return "✅"
+                if has_score or has_cost:
+                    return "⚠️"
+                return "🚫 "
+
+            # Apply the function to create the new column
+            benchmark_table_df['Attempted Benchmark'] = benchmark_table_df.apply(check_benchmark_status, axis=1)
+            # Sort the DataFrame
+            if benchmark_score_col in benchmark_table_df.columns:
+                benchmark_table_df = benchmark_table_df.sort_values(
+                    by=benchmark_score_col, ascending=False, na_position='last'
+                )
+            # 1. Format the cost and score columns
+            benchmark_table_df = format_cost_column(benchmark_table_df, benchmark_cost_col)
+            benchmark_table_df = format_score_column(benchmark_table_df, benchmark_score_col)
+            desired_cols_in_order = [
+                'Agent',
+                'Submitter',
+                'Attempted Benchmark',
+                benchmark_score_col,
+                benchmark_cost_col,
+                'Openness',
+                'Degree of Control',
+                'Date',
+                'Logs'
+            ]
+            for col in desired_cols_in_order:
+                if col not in benchmark_table_df.columns:
+                    benchmark_table_df[col] = pd.NA # Add as an empty column
+            benchmark_table_df = benchmark_table_df[desired_cols_in_order]
+            # Rename columns for a cleaner table display, as requested
+            benchmark_table_df.rename(columns={
+                benchmark_score_col: 'Score',
+                benchmark_cost_col: 'Cost'
+            }, inplace=True)
+            # Ensure the 'Logs' column is formatted correctly
+            table_headers = benchmark_table_df.columns.tolist()
+            # If the column is 'Logs', render as markdown; otherwise, as a string.
+            df_datatypes = [
+                "markdown" if col in ["Logs", "Cost", "Score"] else "str"
+                for col in table_headers
+            ]
+
+            # Create the Gradio component, now with the correct datatypes
+            gr.DataFrame(
+                value=benchmark_table_df,
+                datatype=df_datatypes,
+                interactive=False,
+                wrap=True,
+            )
+
+            # Create the scatter plot using the full data for context, but plotting benchmark metrics
+            # This shows all agents on the same axis for better comparison.
+            benchmark_plot = _plot_scatter_plotly(
+                data=full_df,
+                x=benchmark_cost_col,
+                y=benchmark_score_col,
+                agent_col="Agent"
+            )
+            gr.Plot(value=benchmark_plot)
+            gr.HTML(SCATTER_DISCLAIMER, elem_id="scatter-disclaimer")