|
import json |
|
from datetime import datetime, date |
|
|
|
import gradio as gr |
|
import plotly.graph_objects as go |
|
|
|
|
|
def create_big_five_capex_plot() -> go.Figure: |
|
|
|
with open("big_five_capex.jsonl", "r") as file: |
|
data = [json.loads(line) for line in file if line.strip()] |
|
|
|
quarters: list[str] = [entry["Quarter"] for entry in data] |
|
companies = ['Microsoft', 'Google', 'Meta', 'Amazon'] |
|
colors = ['#80bb00', '#ee161f', '#0065e3', '#ff6200'] |
|
|
|
x_positions = list(range(len(quarters))) |
|
|
|
traces = [] |
|
for company, color in zip(companies, colors): |
|
y_data = [entry[company] for entry in data] |
|
traces.append(go.Bar( |
|
name=company, |
|
x=x_positions, |
|
y=y_data, |
|
marker_color=color |
|
)) |
|
|
|
fig = go.Figure(data=traces) |
|
fig.update_layout( |
|
barmode="stack", |
|
title="Capital Expenditures of Amazon, Meta, Google and Microsoft in Millions of USD per Quarter", |
|
xaxis_title="Quarter", |
|
yaxis_title="Capital Expenditures (Millions USD)", |
|
xaxis=dict( |
|
tickmode='array', |
|
tickvals=x_positions, |
|
ticktext=quarters |
|
), |
|
height=800 |
|
) |
|
|
|
|
|
|
|
try: |
|
idx_q1 = quarters.index("2023 Q1") |
|
idx_q2 = quarters.index("2023 Q2") |
|
vline_x = (idx_q1 + idx_q2) / 2 |
|
except ValueError: |
|
|
|
vline_x = 0 |
|
|
|
|
|
fig.add_shape( |
|
type="line", |
|
xref="x", |
|
yref="paper", |
|
x0=vline_x, |
|
y0=0, |
|
x1=vline_x, |
|
y1=1, |
|
line=dict( |
|
color="black", |
|
dash="dot", |
|
width=2 |
|
) |
|
) |
|
|
|
|
|
fig.add_annotation( |
|
x=vline_x, |
|
y=1.05, |
|
xref="x", |
|
yref="paper", |
|
text="AI arms race begins", |
|
showarrow=False, |
|
font=dict( |
|
color="black", |
|
size=12 |
|
), |
|
align="center" |
|
) |
|
|
|
return fig |
|
|
|
|
|
def create_simple_plot(data_path: str, |
|
name: str, |
|
subtitle: str, |
|
start_date: datetime, end_date: datetime, |
|
min_value: int = 0, max_value: int = 100, |
|
labeled_horizontal_lines: dict[str, float] = None) -> go.Figure: |
|
leaderboard = [] |
|
with open(data_path, 'r') as file: |
|
for line in file: |
|
leaderboard.append(json.loads(line)) |
|
|
|
models = [] |
|
with open("models.jsonl", 'r') as file: |
|
for line in file: |
|
models.append(json.loads(line)) |
|
|
|
data = [] |
|
for entry in leaderboard: |
|
model_name = entry['model'] |
|
score = entry['score'] |
|
model_info = next((m for m in models if m['Name'] == model_name), None) |
|
if model_info: |
|
release_date = datetime.strptime(model_info['Release Date'], "%Y-%m-%d") |
|
data.append({'model': model_name, 'score': score, 'release_date': release_date}) |
|
else: |
|
print(f"[WARNING] Model '{model_name}' not found in models.jsonl") |
|
|
|
data.sort(key=lambda x: x['release_date']) |
|
|
|
x_dates = [d['release_date'] for d in data] |
|
y_scores = [] |
|
max_score = 0 |
|
for entry in data: |
|
if entry['score'] > max_score: |
|
max_score = entry['score'] |
|
y_scores.append(max_score) |
|
|
|
fig = go.Figure() |
|
|
|
fig.add_trace(go.Scatter( |
|
x=x_dates, |
|
y=y_scores, |
|
mode='lines', |
|
line=dict(shape='hv', width=2), |
|
name='Best Score to Date' |
|
)) |
|
|
|
for i, entry in enumerate(data): |
|
if i == 0 or y_scores[i] > y_scores[i - 1]: |
|
fig.add_trace(go.Scatter( |
|
x=[entry['release_date']], |
|
y=[entry['score']], |
|
mode='markers+text', |
|
marker=dict(size=10), |
|
text=[entry['model']], |
|
textposition="top center", |
|
name=entry['model'] |
|
)) |
|
|
|
fig.update_layout( |
|
title=f'{name} Over Time<br><sup>{subtitle}</sup>', |
|
xaxis_title='Publication or Release Date', |
|
yaxis_title=name, |
|
hovermode='x unified', |
|
xaxis=dict( |
|
range=[start_date, end_date], |
|
type='date' |
|
), |
|
yaxis=dict( |
|
range=[min_value, max_value] |
|
), |
|
height=800 |
|
) |
|
|
|
if labeled_horizontal_lines: |
|
for label, y_value in labeled_horizontal_lines.items(): |
|
fig.add_hline( |
|
y=y_value, |
|
line_dash="dot", |
|
line_color="black", |
|
annotation_text=label, |
|
annotation_position="right", |
|
annotation=dict( |
|
font_size=12, |
|
font_color="black", |
|
xanchor="left", |
|
yanchor="middle", |
|
xshift=10 |
|
) |
|
) |
|
|
|
return fig |
|
|
|
|
|
with gr.Blocks() as demo: |
|
with gr.Tab("System Performance Over Time"): |
|
with gr.Tab("Legend"): |
|
legend_markdown: gr.Markdown = gr.Markdown( |
|
value=""" |
|
## Benchmarks and Top Scores |
|
|
|
| Benchmark | Top Score | |
|
|-----------|-----------| |
|
| Humanity's Last Exam | 🔴 7% | |
|
| BigCodeBench | 🟠 36% | |
|
| Simple Bench | 🟠 42% | |
|
| EMMA-Mini | 🟠 48% | |
|
| PlanBench | 🟠 53% | |
|
| NYT Connections | 🟡 60% | |
|
| GAIA | 🟡 65% | |
|
| LiveBench Language | 🟡 65% | |
|
| LiveBench Data Analysis | 🟡 71% | |
|
| LiveCodeBench | 🟡 73% | |
|
| ARC-AGI-Pub (Semi-Private Eval) | 🟡 76% | |
|
| LiveBench | 🟡 76% | |
|
| GPQA | 🟡 76% | |
|
| LiveBench Mathematics | 🟡 81% | |
|
| ZebraLogic | 🟡 81% | |
|
| LiveBench Coding | 🟡 83% | |
|
| ARC-AGI-Pub (Public Eval) | 🟡 83% | |
|
| LiveBench IF | 🟡 86% | |
|
| ZeroEval | 🟡 86% | |
|
| MATH-L5 | 🟡 89% | |
|
| LiveBench Reasoning | 🟢 92% | |
|
| MMLU-Redux | 🟢 93% | |
|
| CRUX | 🟢 96% | |
|
|
|
## Colors |
|
|
|
| Color | Score Range | |
|
|-------|------------| |
|
| 🔴 Red | Below 30% | |
|
| 🟠 Orange | 30% to 60% | |
|
| 🟡 Yellow | 60% to 90% | |
|
| 🟢 Green | Above 90% |""" |
|
) |
|
with gr.Tab("🔴 Humanity's Last Exam") as humanitys_last_exam_tab: |
|
humanitys_last_exam_plot: gr.Plot = gr.Plot() |
|
humanitys_last_exam_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Humanity's Last Exam Quantitative Results](https://lastexam.ai/)""" |
|
) |
|
with gr.Tab("🟠 BigCodeBench") as bigcodebench_tab: |
|
bigcodebench_plot: gr.Plot = gr.Plot() |
|
bigcodebench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [BigCodeBench Leaderboard](https://bigcode-bench.github.io/)""" |
|
) |
|
with gr.Tab("🟠 Simple Bench") as simple_bench_tab: |
|
simple_bench_plot: gr.Plot = gr.Plot() |
|
simple_bench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SimpleBench Leaderboard](https://simple-bench.com/)""" |
|
) |
|
with gr.Tab("🟠 EMMA-Mini") as emma_tab: |
|
emma_plot: gr.Plot = gr.Plot() |
|
emma_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [EMMA Leaderboard](https://emma-benchmark.github.io/#leaderboard)""" |
|
) |
|
with gr.Tab("🟠 PlanBench") as planbench_tab: |
|
planbench_plot: gr.Plot = gr.Plot() |
|
planbench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Valmeekam et al. 2024](https://arxiv.org/abs/2409.13373)""" |
|
) |
|
with gr.Tab("🟡 NYT Connections") as nyt_connections_tab: |
|
nyt_connections_plot: gr.Plot = gr.Plot() |
|
nyt_connections_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [NYT Connections Leaderboard](https://github.com/lechmazur/nyt-connections)""" |
|
) |
|
with gr.Tab("🟡 GAIA") as gaia_tab: |
|
gaia_plot: gr.Plot = gr.Plot() |
|
gaia_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [GAIA Leaderboard](https://huggingface.co/spaces/gaia-benchmark/leaderboard)""" |
|
) |
|
with gr.Tab("🟡 LiveBench Language") as livebench_language_tab: |
|
livebench_language_plot: gr.Plot = gr.Plot() |
|
livebench_language_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟡 LiveBench Data Analysis") as livebench_data_analysis_tab: |
|
livebench_data_analysis_plot: gr.Plot = gr.Plot() |
|
livebench_data_analysis_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟡 LiveCodeBench") as livecodebench_tab: |
|
livecodebench_plot: gr.Plot = gr.Plot() |
|
livecodebench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveCodeBench Leaderboard](https://livecodebench.github.io/leaderboard.html)""" |
|
) |
|
with gr.Tab("🟡 ARC-AGI-Pub") as arc_agi_tab: |
|
with gr.Tab("🟡 Semi-Private Eval") as arc_agi_semi_private_eval_tab: |
|
arc_agi_semi_private_eval_plot: gr.Plot = gr.Plot() |
|
with gr.Tab("🟡 Public Eval") as arc_agi_public_eval_tab: |
|
arc_agi_public_eval_plot: gr.Plot = gr.Plot() |
|
arc_agi_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [ARC Prize 2024](https://arcprize.org/2024-results)""" |
|
) |
|
with gr.Tab("🟡 LiveBench") as livebench_tab: |
|
livebench_plot: gr.Plot = gr.Plot() |
|
livebench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟡 GPQA") as gpqa_tab: |
|
gpqa_plot: gr.Plot = gr.Plot() |
|
gpqa_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Epoch AI Benchmarking Dashboard](https://epoch.ai/data/ai-benchmarking-dashboard)""" |
|
) |
|
with gr.Tab("🟡 LiveBench Mathematics") as livebench_mathematics_tab: |
|
livebench_mathematics_plot: gr.Plot = gr.Plot() |
|
livebench_mathematics_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟡 ZebraLogic") as zeroeval_zebralogic_tab: |
|
zeroeval_zebralogic_plot: gr.Plot = gr.Plot() |
|
zeroeval_zebralogic_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" |
|
) |
|
with gr.Tab("🟡 LiveBench Coding") as livebench_coding_tab: |
|
livebench_coding_plot: gr.Plot = gr.Plot() |
|
livebench_coding_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟡 LiveBench IF") as livebench_if_tab: |
|
livebench_if_plot: gr.Plot = gr.Plot() |
|
livebench_if_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench IF](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟡 ZeroEval") as zeroeval_average_tab: |
|
zeroeval_average_plot: gr.Plot = gr.Plot() |
|
zeroeval_average_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" |
|
) |
|
with gr.Tab("🟡 MATH-L5") as zeroeval_math_l5_tab: |
|
zeroeval_math_l5_plot: gr.Plot = gr.Plot() |
|
zeroeval_math_l5_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" |
|
) |
|
with gr.Tab("🟢 LiveBench Reasoning") as livebench_reasoning_tab: |
|
livebench_reasoning_plot: gr.Plot = gr.Plot() |
|
livebench_reasoning_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" |
|
) |
|
with gr.Tab("🟢 MMLU-Redux") as zeroeval_mmlu_redux_tab: |
|
zeroeval_mmlu_redux_plot: gr.Plot = gr.Plot() |
|
zeroeval_mmlu_redux_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" |
|
) |
|
with gr.Tab("🟢 CRUX") as zeroeval_crux_tab: |
|
zeroeval_crux_plot: gr.Plot = gr.Plot() |
|
zeroeval_crux_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" |
|
) |
|
with gr.Tab("OpenCompass", visible=False): |
|
opencompass_plot: gr.Plot = gr.Plot() |
|
opencompass_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [OpenCompass LLM Leaderboard](https://huggingface.co/spaces/opencompass/opencompass-llm-leaderboard)""" |
|
) |
|
with gr.Tab("SWE-bench", visible=False): |
|
swe_bench_plot: gr.Plot = gr.Plot() |
|
swe_bench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SWE-bench Leaderboard](https://www.swebench.com/)""" |
|
) |
|
with gr.Tab("SWE-bench Multimodal", visible=False): |
|
swe_bench_multimodal_plot: gr.Plot = gr.Plot() |
|
swe_bench_multimodal_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SWE-bench Leaderboard](https://www.swebench.com/#multimodal)""" |
|
) |
|
with gr.Tab("WebArena", visible=False): |
|
webarena_plot: gr.Plot = gr.Plot() |
|
webarena_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [X-WebArena-Leaderboard](https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ)""" |
|
) |
|
with gr.Tab("OSWorld", visible=False): |
|
osworld_plot: gr.Plot = gr.Plot() |
|
osworld_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [OSWorld Benchmark](https://os-world.github.io/)""" |
|
) |
|
with gr.Tab("MathVista", visible=False): |
|
mathvista_plot: gr.Plot = gr.Plot() |
|
mathvista_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Leaderboard on MathVista](https://mathvista.github.io/#leaderboard)""" |
|
) |
|
with gr.Tab("DABStep", visible=False): |
|
dabstep_plot: gr.Plot = gr.Plot() |
|
dabstep_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [DABStep Leaderboard](https://huggingface.co/spaces/adyen/DABstep)""" |
|
) |
|
with gr.Tab("lineage-bench", visible=False): |
|
lineage_bench_plot: gr.Plot = gr.Plot() |
|
lineage_bench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [lineage-bench Results](https://github.com/fairydreaming/lineage-bench)""" |
|
) |
|
with gr.Tab("Step-Game", visible=False): |
|
step_game_plot: gr.Plot = gr.Plot() |
|
step_game_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Step-Game TrueSkill Leaderboard](https://github.com/lechmazur/step_game)""" |
|
) |
|
with gr.Tab("HHEM", visible=False): |
|
hhem_plot: gr.Plot = gr.Plot() |
|
hhem_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Vectara Hallucination Leaderboard](https://github.com/vectara/hallucination-leaderboard)""" |
|
) |
|
with gr.Tab("USACO", visible=False): |
|
usaco_plot: gr.Plot = gr.Plot() |
|
usaco_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [USACO Leaderboard](https://hal.cs.princeton.edu/usaco)""" |
|
) |
|
with gr.Tab("AppWorld", visible=False): |
|
appworld_plot: gr.Plot = gr.Plot() |
|
appworld_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [AppWorld Agent Scores](https://appworld.dev/leaderboard)""" |
|
) |
|
with gr.Tab("CORE-Bench", visible=False): |
|
core_bench_plot: gr.Plot = gr.Plot() |
|
core_bench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [HAL Leaderboards](https://hal.cs.princeton.edu/#leaderboards)""" |
|
) |
|
with gr.Tab("Cybench", visible=False): |
|
cybench_plot: gr.Plot = gr.Plot() |
|
cybench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Cybench Leaderboard](https://hal.cs.princeton.edu/cybench)""" |
|
) |
|
with gr.Tab("MultiChallenge", visible=False): |
|
multichallenge_plot: gr.Plot = gr.Plot() |
|
multichallenge_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SEAL Leaderboard: MultiChallenge](https://scale.com/leaderboard/multichallenge)""" |
|
) |
|
with gr.Tab("VISTA", visible=False): |
|
vista_plot: gr.Plot = gr.Plot() |
|
vista_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SEAL Leaderboard: Visual-Language Understanding](https://scale.com/leaderboard/visual_language_understanding)""" |
|
) |
|
with gr.Tab("ToolComp", visible=False): |
|
with gr.Tab("Enterprise"): |
|
toolcomp_enterprise_plot: gr.Plot = gr.Plot() |
|
toolcomp_enterprise_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SEAL Leaderboard: Agentic Tool Use (Enterprise)](https://scale.com/leaderboard/tool_use_enterprise)""" |
|
) |
|
with gr.Tab("Chat"): |
|
toolcomp_chat_plot: gr.Plot = gr.Plot() |
|
toolcomp_chat_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [SEAL Leaderboard: Agentic Tool Use (Chat)](https://scale.com/leaderboard/tool_use_chat)""" |
|
) |
|
with gr.Tab("BFCL", visible=False): |
|
bfcl_plot: gr.Plot = gr.Plot() |
|
bfcl_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [BFCL Leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html)""" |
|
) |
|
with gr.Tab("EvalPlus", visible=False): |
|
evalplus_plot: gr.Plot = gr.Plot() |
|
evalplus_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [EvalPlus Leaderboard](https://evalplus.github.io/leaderboard.html)""" |
|
) |
|
with gr.Tab("Aider Polyglot", visible=False): |
|
aider_plot: gr.Plot = gr.Plot() |
|
aider_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [Aider LLM Leaderboards](https://aider.chat/docs/leaderboards/)""" |
|
) |
|
with gr.Tab("QuALITY", visible=False): |
|
quality_plot: gr.Plot = gr.Plot() |
|
quality_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [QuALITY Leaderboard](https://nyu-mll.github.io/quality/)""" |
|
) |
|
with gr.Tab("MMVU", visible=False): |
|
mmvu_plot: gr.Plot = gr.Plot() |
|
mmvu_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [MMVU Leaderboard](https://mmvu-benchmark.github.io/#leaderboard)""" |
|
) |
|
with gr.Tab("PhysBench", visible=False): |
|
physbench_plot: gr.Plot = gr.Plot() |
|
physbench_markdown: gr.Markdown = gr.Markdown( |
|
value="""Source: [PhysBench Leaderboard](https://physbench.github.io/#leaderboard)""" |
|
) |
|
with gr.Tab("Finance") as finance_tab: |
|
with gr.Tab("Big Tech Capex") as big_five_capex_tab: |
|
big_five_capex_plot: gr.Plot = gr.Plot() |
|
with gr.Tab("NVIDIA Revenue", visible=False) as nvidia_revenue: |
|
nvidia_revenue_plot: gr.Plot = gr.Plot() |
|
big_five_capex_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot) |
|
arc_agi_public_eval_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("arc_agi_leaderboard.jsonl"), |
|
gr.State( |
|
"ARC-AGI-Pub Score (Public Eval, $20 Compute Budget per Task, General-Purpose Systems)"), |
|
gr.State( |
|
"\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"), |
|
gr.State(date(2024, 6, 20)), gr.State(date(2025, 1, 1)), |
|
gr.State(0), gr.State(100), |
|
gr.State({"Humans\n(LeGris et al. 2024)": 64.2})], |
|
outputs=arc_agi_public_eval_plot) |
|
arc_agi_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("arc_agi_semi_private_eval_leaderboard.jsonl"), |
|
gr.State( |
|
"ARC-AGI-Pub Score (Semi-Private Eval, $20 Compute Budget per Task, General-Purpose Systems)"), |
|
gr.State( |
|
"\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"), |
|
gr.State(date(2024, 6, 20)), gr.State(date(2025, 1, 1)), |
|
gr.State(0), gr.State(100), |
|
gr.State({"MTurkers": 77})], |
|
outputs=arc_agi_semi_private_eval_plot) |
|
arc_agi_semi_private_eval_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("arc_agi_semi_private_eval_leaderboard.jsonl"), |
|
gr.State( |
|
"ARC-AGI-Pub Score (Semi-Private Eval, $20 Compute Budget per Task, General-Purpose Systems)"), |
|
gr.State( |
|
"\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"), |
|
gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)), |
|
gr.State(0), gr.State(100), |
|
gr.State({"MTurkers": 77})], |
|
outputs=arc_agi_semi_private_eval_plot) |
|
finance_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot) |
|
simple_bench_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("simple_bench_leaderboard.jsonl"), |
|
gr.State("Simple Bench Score"), |
|
gr.State( |
|
"\"multiple-choice text benchmark [...] [including] over 200 questions covering spatio-temporal reasoning, social intelligence, and what we call linguistic adversarial robustness\" (Philip & Hemang, 2024)"), |
|
gr.State(date(2024, 4, 9)), gr.State(date(2025, 2, 1)), |
|
gr.State(0), gr.State(100), |
|
gr.State({"Humans": 83.7})], |
|
outputs=simple_bench_plot) |
|
planbench_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("planbench_leaderboard.jsonl"), |
|
gr.State("PlanBench Score (Mystery Blocksworld, 0-shot)"), |
|
gr.State( |
|
"\"benchmark suite based on the kinds of domains used in the automated planning community [...] to test the capabilities of LLMs in planning or reasoning about actions and change.\" (Valmeekam et al. 2022)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2024, 9, 20))], |
|
outputs=planbench_plot) |
|
bigcodebench_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("bigcodebench_hard_average_leaderboard.jsonl"), |
|
gr.State("BigCodeBench Score (Hard, Average of Complete and Instruct)"), |
|
gr.State( |
|
"\"benchmark that challenges LLMs to invoke multiple function calls as tools from 139 libraries and 7 domains for 1,140 fine-grained tasks\" (Zhuo et al. 2024)"), |
|
gr.State(date(2023, 6, 1)), gr.State(date(2025, 1, 1))], |
|
outputs=bigcodebench_plot) |
|
gaia_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("gaia_leaderboard.jsonl"), |
|
gr.State("General AI Assistants (GAIA) Benchmark Score (Test Set, Average)"), |
|
gr.State( |
|
"\"real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency\" (Mialon et al. 2023)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1)), |
|
gr.State(0), gr.State(100), |
|
gr.State({"Humans": 92})], |
|
outputs=gaia_plot) |
|
gpqa_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("gpqa_leaderboard.jsonl"), |
|
gr.State("Graduate-Level Google-Proof Q&A (GPQA) Benchmark Score"), |
|
gr.State( |
|
"\"challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry [that] are high-quality and extremely difficult\" (Rein et al. 2023)"), |
|
gr.State(date(2023, 6, 1)), gr.State(date(2025, 1, 1)), |
|
gr.State(25), gr.State(100), |
|
gr.State({"Highly skilled non-expert validators": 34, "PhD-level domain experts": 65})], |
|
outputs=gpqa_plot) |
|
zeroeval_average_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("zeroeval_average_leaderboard.jsonl"), |
|
gr.State("ZeroEval Average (MMLU-Redux, ZebraLogic, CRUX and MATH-5) Score"), |
|
gr.State( |
|
"\"a simple unified framework for evaluating language models on various tasks\" (Ai2, 2024)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], |
|
outputs=zeroeval_average_plot) |
|
zeroeval_mmlu_redux_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("zeroeval_mmlu_redux_leaderboard.jsonl"), |
|
gr.State( |
|
"ZeroEval MMLU-Redux (Massive Multitask Language Understanding) Score"), |
|
gr.State( |
|
"\"knowledge reasoning\" (Ai2, 2024); \"subset of 3,000 manually re-annotated questions across 30 MMLU subjects\" (Gema et al. 2024)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], |
|
outputs=zeroeval_mmlu_redux_plot) |
|
zeroeval_zebralogic_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("zeroeval_zebralogic_leaderboard.jsonl"), |
|
gr.State("ZeroEval ZebraLogic Score"), |
|
gr.State( |
|
"\"logical reasoning\" (Ai2, 2024); \"Each example is a Logic Grid Puzzle [...] often used to test humans' logical reasoning abilities\" (Lin, 2024)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], |
|
outputs=zeroeval_zebralogic_plot) |
|
zeroeval_crux_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("zeroeval_crux_leaderboard.jsonl"), |
|
gr.State( |
|
"ZeroEval CRUX (Code Reasoning, Understanding, and eXecution Evaluation) Score"), |
|
gr.State( |
|
"\"code reasoning\" (Ai2, 2024); \"benchmark consisting of 800 Python functions (3-13 lines). Each function comes with [...] two natural tasks: input prediction and output prediction.\" (Gu et al. 2024)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], |
|
outputs=zeroeval_crux_plot) |
|
zeroeval_math_l5_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("zeroeval_math_l5_leaderboard.jsonl"), |
|
gr.State("ZeroEval MATH-L5 (Difficulty Level 5 of MATH) Score"), |
|
gr.State( |
|
"\"math reasoning\" (Ai2, 2024); \"dataset of 12,500 challenging competition mathematics problems. [...] a subject’s hardest problems are assigned a difficulty level of ‘5.’\" (Hendrycks et al. 2021)"), |
|
gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], |
|
outputs=zeroeval_math_l5_plot) |
|
livebench_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench.jsonl"), |
|
gr.State("LiveBench-2024-11-25: Global Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_plot) |
|
livebench_reasoning_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench_reasoning.jsonl"), |
|
gr.State("LiveBench-2024-11-25: Reasoning Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_reasoning_plot) |
|
livebench_coding_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench_coding.jsonl"), |
|
gr.State("LiveBench-2024-11-25: Coding Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_coding_plot) |
|
livebench_mathematics_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench_mathematics.jsonl"), |
|
gr.State("LiveBench-2024-11-25: Mathematics Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_mathematics_plot) |
|
livebench_data_analysis_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench_data_analysis.jsonl"), |
|
gr.State("LiveBench-2024-11-25: Data Analysis Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_data_analysis_plot) |
|
livebench_language_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench_language.jsonl"), |
|
gr.State("LiveBench-2024-11-25: Language Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_language_plot) |
|
livebench_if_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livebench_if.jsonl"), |
|
gr.State("LiveBench-2024-11-25: IF Average Score"), |
|
gr.State( |
|
"\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), |
|
gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], |
|
outputs=livebench_if_plot) |
|
humanitys_last_exam_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("humanitys_last_exam.jsonl"), |
|
gr.State("Humanity's Last Exam (Multi-Modal Models Only) Score"), |
|
gr.State( |
|
"\"multi-modal benchmark at the frontier of human knowledge, designed to be the final closed-ended academic benchmark of its kind with broad subject coverage\" (Phan et al. 2025)"), |
|
gr.State(date(2024, 5, 13)), gr.State(date(2025, 2, 11))], |
|
outputs=humanitys_last_exam_plot) |
|
livecodebench_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("livecodebench.jsonl"), |
|
gr.State("LiveCodeBench (7/1/2024 to 2/1/2025) Score"), |
|
gr.State( |
|
"\"comprehensive and contamination-free evaluation of LLMs for code, which continuously collects new problems over time from contests across three competition platforms\" (Jain et al. 2024)"), |
|
gr.State(date(2024, 4, 9)), gr.State(date(2025, 2, 1))], |
|
outputs=livecodebench_plot) |
|
emma_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("emma_mini.jsonl"), |
|
gr.State("EMMA-Mini (Enhanced MultiModal ReAsoning) Score"), |
|
gr.State("\"benchmark targeting organic multimodal reasoning across mathematics, physics, chemistry, and coding\" (Hao et al. 2025)"), |
|
gr.State(date(2024, 9, 17)), gr.State(date(2025, 2, 1)), |
|
gr.State(22.75), gr.State(100), |
|
gr.State({"Human experts": 77.75})], |
|
outputs=emma_plot) |
|
nyt_connections_tab.select(fn=create_simple_plot, |
|
inputs=[gr.State("nyt_connections.jsonl"), |
|
gr.State("NYT Connections (Extended Version, Newest 100 Puzzles) Score"), |
|
gr.State("\"NYT Connections puzzles [...] To increase difficulty, Extended Connections adds up to four extra trick words to each puzzle.\" (Mazur, 2025)"), |
|
gr.State(date(2024, 7, 23)), gr.State(date(2025, 2, 1))], |
|
outputs=nyt_connections_plot) |
|
|
|
if __name__ == "__main__": |
|
demo.launch() |
|
|
