app.py

import argparse
import ast
import glob
import pickle
import traceback
from datetime import datetime

import pandas as pd
import gradio as gr
import numpy as np


basic_component_values = [None] * 6
leader_component_values = [None] * 5

def make_default_md_1():
    leaderboard_md = f"""
# 🏆 LLM Arena in Russian: Leaderboard
"""
    return leaderboard_md


def make_default_md_2():
    leaderboard_md = f"""
    
    The LLM Arena platform is an open crowdsourcing platform for evaluating large language models (LLM) in Russian. We collect pairwise comparisons from people to rank LLMs using the Bradley-Terry model and display model ratings on the Elo scale.
    Chatbot Arena in Russian depends on community participation, so please contribute by casting your vote!

    - To **add your model** to the comparison, contact us on TG: [Group](https://t.me/+bFEOl-Bdmok4NGUy)
    - If you **found a bug** or **have a suggestion**, contact us: [Roman](https://t.me/roman_kucev)
    - You can contribute your vote at llmarena.ru!
    """

    return leaderboard_md



def make_arena_leaderboard_md(arena_df, last_updated_time):
    total_votes = sum(arena_df["num_battles"])
    total_models = len(arena_df)
    space = "   "

    leaderboard_md = f"""
Total # of models: **{total_models}**.{space} Total # of votes: **{"{:,}".format(total_votes)}**.{space} Last updated: {last_updated_time}.

***Rank (UB)**: model rating (upper bound), determined as one plus the number of models that are statistically better than the target model.
Model A is statistically better than Model B when the lower bound of Model A's rating is higher than the upper bound of Model B's rating (with a 95% confidence interval).
See Figure 1 below for a visualization of the confidence intervals of model ratings.
"""
    return leaderboard_md



def make_category_arena_leaderboard_md(arena_df, arena_subset_df, name="Overall"):
    total_votes = sum(arena_df["num_battles"])
    total_models = len(arena_df)
    space = "   "
    total_subset_votes = sum(arena_subset_df["num_battles"])
    total_subset_models = len(arena_subset_df)
    leaderboard_md = f"""### {cat_name_to_explanation[name]}
#### {space} #models: **{total_subset_models} ({round(total_subset_models / total_models * 100)}%)** {space} #votes: **{"{:,}".format(total_subset_votes)} ({round(total_subset_votes / total_votes * 100)}%)**{space}
"""
    return leaderboard_md



def model_hyperlink(model_name, link):
    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'


def load_leaderboard_table_csv(filename, add_hyperlink=True):
    lines = open(filename).readlines()
    heads = [v.strip() for v in lines[0].split(",")]
    rows = []
    for i in range(1, len(lines)):
        row = [v.strip() for v in lines[i].split(",")]
        for j in range(len(heads)):
            item = {}
            for h, v in zip(heads, row):
                if h == "Arena Elo rating":
                    if v != "-":
                        v = int(ast.literal_eval(v))
                    else:
                        v = np.nan
                elif h == "MMLU":
                    if v != "-":
                        v = round(ast.literal_eval(v) * 100, 1)
                    else:
                        v = np.nan
                elif h == "MT-bench (win rate %)":
                    if v != "-":
                        v = round(ast.literal_eval(v[:-1]), 1)
                    else:
                        v = np.nan
                elif h == "MT-bench (score)":
                    if v != "-":
                        v = round(ast.literal_eval(v), 2)
                    else:
                        v = np.nan
                item[h] = v
            if add_hyperlink:
                item["Model"] = model_hyperlink(item["Model"], item["Link"])
        rows.append(item)

    return rows


def create_ranking_str(ranking, ranking_difference):
    if ranking_difference > 0:
        return f"{int(ranking)} \u2191"
    elif ranking_difference < 0:
        return f"{int(ranking)} \u2193"
    else:
        return f"{int(ranking)}"


def recompute_final_ranking(arena_df):
    # compute ranking based on CI
    ranking = {}
    for i, model_a in enumerate(arena_df.index):
        ranking[model_a] = 1
        for j, model_b in enumerate(arena_df.index):
            if i == j:
                continue
            if (
                arena_df.loc[model_b]["rating_q025"]
                > arena_df.loc[model_a]["rating_q975"]
            ):
                ranking[model_a] += 1
    return list(ranking.values())


def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
    arena_df = arena_df.sort_values(
        by=["final_ranking", "rating"], ascending=[True, False]
    )
    arena_df["final_ranking"] = recompute_final_ranking(arena_df)
    arena_df = arena_df.sort_values(
        by=["final_ranking", "rating"], ascending=[True, False]
    )

    # sort by rating
    if arena_subset_df is not None:
        # filter out models not in the arena_df
        arena_subset_df = arena_subset_df[arena_subset_df.index.isin(arena_df.index)]
        arena_subset_df = arena_subset_df.sort_values(by=["rating"], ascending=False)
        arena_subset_df["final_ranking"] = recompute_final_ranking(arena_subset_df)
        # keep only the models in the subset in arena_df and recompute final_ranking
        arena_df = arena_df[arena_df.index.isin(arena_subset_df.index)]
        # recompute final ranking
        arena_df["final_ranking"] = recompute_final_ranking(arena_df)

        # assign ranking by the order
        arena_subset_df["final_ranking_no_tie"] = range(1, len(arena_subset_df) + 1)
        arena_df["final_ranking_no_tie"] = range(1, len(arena_df) + 1)
        # join arena_df and arena_subset_df on index
        arena_df = arena_subset_df.join(
            arena_df["final_ranking"], rsuffix="_global", how="inner"
        )
        arena_df["ranking_difference"] = (
            arena_df["final_ranking_global"] - arena_df["final_ranking"]
        )

        arena_df = arena_df.sort_values(
            by=["final_ranking", "rating"], ascending=[True, False]
        )
        arena_df["final_ranking"] = arena_df.apply(
            lambda x: create_ranking_str(x["final_ranking"], x["ranking_difference"]),
            axis=1,
        )

    arena_df["final_ranking"] = arena_df["final_ranking"].astype(str)

    values = []
    for i in range(len(arena_df)):
        row = []
        model_key = arena_df.index[i]
        try:
            model_name = model_table_df[model_table_df["key"] == model_key][
                "Model"
            ].values[0]
            ranking = arena_df.iloc[i].get("final_ranking") or i + 1
            row.append(ranking)
            if arena_subset_df is not None:
                row.append(arena_df.iloc[i].get("ranking_difference") or 0)
            row.append(model_name)
            row.append(round(arena_df.iloc[i]["rating"]))
            upper_diff = round(
                arena_df.iloc[i]["rating_q975"] - arena_df.iloc[i]["rating"]
            )
            lower_diff = round(
                arena_df.iloc[i]["rating"] - arena_df.iloc[i]["rating_q025"]
            )
            row.append(f"+{upper_diff}/-{lower_diff}")
            row.append(round(arena_df.iloc[i]["num_battles"]))
            row.append(
                model_table_df[model_table_df["key"] == model_key][
                    "Organization"
                ].values[0]
            )
            row.append(
                model_table_df[model_table_df["key"] == model_key]["License"].values[0]
            )
            cutoff_date = model_table_df[model_table_df["key"] == model_key][
                "Knowledge cutoff date"
            ].values[0]
            if cutoff_date == "-":
                row.append("Unknown")
            else:
                row.append(cutoff_date)
            values.append(row)
        except Exception as e:
            traceback.print_exc()
            print(f"{model_key} - {e}")
    return values


key_to_category_name = {
    "full": "Overall",
    "crowdsourcing/simple_prompts": "crowdsourcing/simple_prompts",
    "site_visitors/medium_prompts": "site_visitors/medium_prompts",
    "site_visitors/medium_prompts:style control": "site_visitors/medium_prompts:style control"
}
cat_name_to_explanation = {
    "Overall": "All queries",
    "crowdsourcing/simple_prompts": "Queries collected through crowdsourcing. Mostly simple ones.",
    "site_visitors/medium_prompts": "Queries from website visitors. Contain more complex prompts.",
    "site_visitors/medium_prompts:style control": "Queries from website visitors. Contain more complex prompts. [Reduced stylistic influence](https://lmsys.org/blog/2024-08-28-style-control/) of the response on the rating."
}

cat_name_to_baseline = {
    "Hard Prompts (English)": "English",
}

actual_categories = [
    "Overall",
    "crowdsourcing/simple_prompts",
    "site_visitors/medium_prompts",
    "site_visitors/medium_prompts:style control"
]


def read_elo_file(elo_results_file, leaderboard_table_file):
    arena_dfs = {}
    category_elo_results = {}
    with open(elo_results_file, "rb") as fin:
        elo_results = pickle.load(fin)
        last_updated_time = None
        if "full" in elo_results:
            last_updated_time = elo_results["full"]["last_updated_datetime"].split(
                " "
            )[0]
            for k in key_to_category_name.keys():
                if k not in elo_results:
                    continue
                arena_dfs[key_to_category_name[k]] = elo_results[k][
                    "leaderboard_table_df"
                ]
                category_elo_results[key_to_category_name[k]] = elo_results[k]

    data = load_leaderboard_table_csv(leaderboard_table_file)
        

    model_table_df = pd.DataFrame(data)

    return last_updated_time, arena_dfs, category_elo_results, elo_results, model_table_df


def build_leaderboard_tab(
    elo_results_file, leaderboard_table_file, show_plot=False, mirror=False
):
    arena_dfs = {}
    arena_df = pd.DataFrame()
    category_elo_results = {}
        
    last_updated_time, arena_dfs, category_elo_results, elo_results, model_table_df = read_elo_file(elo_results_file, leaderboard_table_file)

    p1 = category_elo_results["Overall"]["win_fraction_heatmap"]
    p2 = category_elo_results["Overall"]["battle_count_heatmap"]
    p3 = category_elo_results["Overall"]["bootstrap_elo_rating"]
    p4 = category_elo_results["Overall"]["average_win_rate_bar"]
    arena_df = arena_dfs["Overall"]
    default_md = make_default_md_1()
    default_md_2 = make_default_md_2()
    
    with gr.Row():
        with gr.Column(scale=4):
            md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
        with gr.Column(scale=1):
            vote_button = gr.Button("Vote!", link="https://llmarena.ru")
    md_2 = gr.Markdown(default_md_2, elem_id="leaderboard_markdown")
    
    if leaderboard_table_file:
        data = load_leaderboard_table_csv(leaderboard_table_file)

        model_table_df = pd.DataFrame(data)

        with gr.Tabs() as tabs:
            arena_table_vals = get_arena_table(arena_df, model_table_df)

            with gr.Tab("Арена", id=0):
                md = make_arena_leaderboard_md(arena_df, last_updated_time)

                lb_description = gr.Markdown(md, elem_id="leaderboard_markdown")
                with gr.Row():
                    with gr.Column(scale=2):
                        category_dropdown = gr.Dropdown(
                            choices=actual_categories,
                            label="Category",
                            value="Overall",
                        )
                    default_category_details = make_category_arena_leaderboard_md(
                            arena_df, arena_df, name="Overall"
                        )

                    with gr.Column(scale=4, variant="panel"):
                        category_deets = gr.Markdown(
                            default_category_details, elem_id="category_deets"
                        )

                arena_vals = pd.DataFrame(
                    arena_table_vals,
                    columns=[
                        "Rank* (UB)",
                        "Model",
                        "Arena Elo",
                        "95% CI",
                        "Votes",
                        "Organization",
                        "License",
                        "Knowledge Cutoff",
                    ],
                )
                elo_display_df = gr.Dataframe(
                    headers=[
                        "Rank* (UB)",
                        "Model",
                        "Arena Elo",
                        "95% CI",
                        "Votes",
                        "Organization",
                        "License",
                        "Knowledge Cutoff",
                    ],
                    datatype=[
                        "str",
                        "markdown",
                        "number",
                        "str",
                        "number",
                        "str",
                        "str",
                        "str",
                    ],
                    value=arena_vals.style,
                    elem_id="arena_leaderboard_dataframe",
                    height=700,
                    column_widths=[70, 190, 100, 100, 90, 130, 150, 100],
                    wrap=True,
                )

                gr.Markdown(
                    elem_id="leaderboard_markdown",
                )

                leader_component_values[:] = [default_md, p1, p2, p3, p4]

                if show_plot:
                    more_stats_md = gr.Markdown(
                        f"""## More statistics on Chatbot Arena""",
                        elem_id="leaderboard_header_markdown",
                    )
                    with gr.Row():
                        with gr.Column():
                            gr.Markdown(
                                "#### Figure 1: Confidence Intervals on Model Strength (via Bootstrapping)",
                                elem_id="plot-title",
                            )
                            plot_3 = gr.Plot(p3, show_label=False)
                        with gr.Column():
                            gr.Markdown(
                                "#### Figure 2: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)",
                                elem_id="plot-title",
                            )
                            plot_4 = gr.Plot(p4, show_label=False)
                    with gr.Row():
                        with gr.Column():
                            gr.Markdown(
                                "#### Figure 3: Fraction of Model A Wins for All Non-tied A vs. B Battles",
                                elem_id="plot-title",
                            )
                            plot_1 = gr.Plot(
                                p1, show_label=False, elem_id="plot-container"
                            )
                        with gr.Column():
                            gr.Markdown(
                                "#### Figure 4: Battle Count for Each Combination of Models (without Ties)",
                                elem_id="plot-title",
                            )
                            plot_2 = gr.Plot(p2, show_label=False)

        if not show_plot:
            gr.Markdown(
                """
                """,
                elem_id="leaderboard_markdown",
            )
    else:
        pass

    def update_leaderboard_df(arena_table_vals):
        elo_datarame = pd.DataFrame(
            arena_table_vals,
            columns=[
                "Rank* (UB)",
                "Delta",
                "Model",
                "Arena Elo",
                "95% CI",
                "Votes",
                "Organization",
                "License",
                "Knowledge Cutoff",
            ],
        )

        def highlight_max(s):
            return [
                "color: green; font-weight: bold"
                if "\u2191" in v
                else "color: red; font-weight: bold"
                if "\u2193" in v
                else ""
                for v in s
            ]

        def highlight_rank_max(s):
            return [
                "color: green; font-weight: bold"
                if v > 0
                else "color: red; font-weight: bold"
                if v < 0
                else ""
                for v in s
            ]

        return elo_datarame.style.apply(highlight_max, subset=["Rank* (UB)"]).apply(
            highlight_rank_max, subset=["Delta"]
        )

    def update_leaderboard_and_plots(category):
        _, arena_dfs, category_elo_results, _ , model_table_df = read_elo_file(elo_results_file, leaderboard_table_file)

        arena_subset_df = arena_dfs[category]
        arena_subset_df = arena_subset_df[arena_subset_df["num_battles"] > 300]
        elo_subset_results = category_elo_results[category]

        baseline_category = cat_name_to_baseline.get(category, "Overall")
        arena_df = arena_dfs[baseline_category]
        arena_values = get_arena_table(
            arena_df,
            model_table_df,
            arena_subset_df=arena_subset_df if category != "Overall" else None,
        )
        if category != "Overall":
            arena_values = update_leaderboard_df(arena_values)
            arena_values = gr.Dataframe(
                headers=[
                    "Rank* (UB)",
                    "Delta",
                    "Model",
                    "Arena Elo",
                    "95% CI",
                    "Votes",
                    "Organization",
                    "License",
                    "Knowledge Cutoff",
                ],
                datatype=[
                    "str",
                    "number",
                    "markdown",
                    "number",
                    "str",
                    "number",
                    "str",
                    "str",
                    "str",
                ],
                value=arena_values,
                elem_id="arena_leaderboard_dataframe",
                height=700,
                column_widths=[70, 70, 200, 90, 100, 90, 120, 150, 100],
                wrap=True,
            )
        else:
            arena_values = gr.Dataframe(
                headers=[
                    "Rank* (UB)",
                    "Model",
                    "Arena Elo",
                    "95% CI",
                    "Votes",
                    "Organization",
                    "License",
                    "Knowledge Cutoff",
                ],
                datatype=[
                    "str",
                    "markdown",
                    "number",
                    "str",
                    "number",
                    "str",
                    "str",
                    "str",
                ],
                value=arena_values,
                elem_id="arena_leaderboard_dataframe",
                height=700,
                column_widths=[70, 190, 100, 100, 90, 140, 150, 100],
                wrap=True,
            )

        p1 = elo_subset_results["win_fraction_heatmap"]
        p2 = elo_subset_results["battle_count_heatmap"]
        p3 = elo_subset_results["bootstrap_elo_rating"]
        p4 = elo_subset_results["average_win_rate_bar"]
        more_stats_md = f"""## More Statistics for Chatbot Arena - {category}
        """
        leaderboard_md = make_category_arena_leaderboard_md(
            arena_df, arena_subset_df, name=category
        )
        return arena_values, p1, p2, p3, p4, more_stats_md, leaderboard_md

    if leaderboard_table_file:
        category_dropdown.change(
            fn=update_leaderboard_and_plots,
            inputs=[category_dropdown],
            outputs=[
                elo_display_df,
                plot_1,
                plot_2,
                plot_3,
                plot_4,
                more_stats_md,
                category_deets,
            ],
        )
    if show_plot and leaderboard_table_file:
        return [md_1, md_2, lb_description, category_deets, elo_display_df, plot_1, plot_2, plot_3, plot_4]
    return [md_1]


def build_demo(elo_results_file, leaderboard_table_file):
    text_size = gr.themes.sizes.text_lg
    theme = gr.themes.Default.load("theme.json")
    theme.text_size = text_size
    theme.set(
        button_large_text_size="40px",
        button_small_text_size="40px",
        button_large_text_weight="1000",
        button_small_text_weight="1000",
        button_shadow="*shadow_drop_lg",
        button_shadow_hover="*shadow_drop_lg",
        checkbox_label_shadow="*shadow_drop_lg",
        button_shadow_active="*shadow_inset",
        button_secondary_background_fill="*primary_300",
        button_secondary_background_fill_dark="*primary_700",
        button_secondary_background_fill_hover="*primary_200",
        button_secondary_background_fill_hover_dark="*primary_500",
        button_secondary_text_color="*primary_800",
        button_secondary_text_color_dark="white",
    )

    with gr.Blocks(
        title="LLM arena: leaderboard",
        theme=theme,
        css=block_css,
    ) as demo:
        build_leaderboard_tab(
            elo_results_file, leaderboard_table_file, show_plot=True, mirror=True
        )
    return demo

block_css = """
#notice_markdown .prose {
    font-size: 110% !important;
}
#notice_markdown th {
    display: none;
}
#notice_markdown td {
    padding-top: 6px;
    padding-bottom: 6px;
}
#arena_leaderboard_dataframe table {
    font-size: 110%;
}
#full_leaderboard_dataframe table {
    font-size: 110%;
}
#model_description_markdown {
    font-size: 110% !important;
}
#leaderboard_markdown .prose {
    font-size: 110% !important;
}
#leaderboard_markdown td {
    padding-top: 6px;
    padding-bottom: 6px;
}
#leaderboard_dataframe td {
    line-height: 0.1em;
}
#about_markdown .prose {
    font-size: 110% !important;
}
#ack_markdown .prose {
    font-size: 110% !important;
}
#chatbot .prose {
    font-size: 105% !important;
}
.sponsor-image-about img {
    margin: 0 20px;
    margin-top: 20px;
    height: 40px;
    max-height: 100%;
    width: auto;
    float: left;
}

.chatbot h1, h2, h3 {
    margin-top: 8px; /* Adjust the value as needed */
    margin-bottom: 0px; /* Adjust the value as needed */
    padding-bottom: 0px;
}

.chatbot h1 {
    font-size: 130%;
}
.chatbot h2 {
    font-size: 120%;
}
.chatbot h3 {
    font-size: 110%;
}
.chatbot p:not(:first-child) {
    margin-top: 8px;
}

.typing {
    display: inline-block;
}

.cursor {
    display: inline-block;
    width: 7px;
    height: 1em;
    background-color: black;
    vertical-align: middle;
    animation: blink 1s infinite;
}

.dark .cursor {
    display: inline-block;
    width: 7px;
    height: 1em;
    background-color: white;
    vertical-align: middle;
    animation: blink 1s infinite;
}

@keyframes blink {
    0%, 50% { opacity: 1; }
    50.1%, 100% { opacity: 0; }
}

.app {
  max-width: 100% !important;
  padding: 20px !important;               
}

a {
    color: #1976D2; /* Your current link color, a shade of blue */
    text-decoration: none; /* Removes underline from links */
}
a:hover {
    color: #63A4FF; /* This can be any color you choose for hover */
    text-decoration: underline; /* Adds underline on hover */
}
"""


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--share", action="store_true")
    parser.add_argument("--host", default="0.0.0.0")
    parser.add_argument("--port", type=int, default=7860)
    args = parser.parse_args()

    elo_result_files = glob.glob("elo_results_*.pkl")
    elo_result_files.sort(key=lambda x: int(x[12:-4]))
    elo_result_file = elo_result_files[-1]

    leaderboard_table_files = glob.glob("leaderboard_table_*.csv")
    leaderboard_table_files.sort(key=lambda x: int(x[18:-4]))
    leaderboard_table_file = leaderboard_table_files[-1]

    demo = build_demo(elo_result_file, leaderboard_table_file)
    demo.launch(show_api=False)