update UI

app.py

@@ -13,6 +13,7 @@ from huggingface_hub import HfApi
 from huggingface_hub.hf_api import HTTPError
 from huggingface_hub.utils._errors import GatedRepoError
 from gradio_rangeslider import RangeSlider
+import datetime
 
 
 load_dotenv()
@@ -139,7 +140,7 @@ def get_unique_column_names(all_data):
         "arxiv_\u200bphysics",
         "github_\u200bcpp",
         "github_\u200bpython",
-        "ao3_\u200bchinese",
+        # "ao3_\u200bchinese",
     ]
 
 
@@ -149,74 +150,98 @@ def color_cell(value):
 
 def update_table(
     period: str,
-    models: list,
+    models_size: list,
     metric: str,
     visible_columns: list,
     color_columns: list,
     size_range: list,
-    sort_by: str = "Average (The lower the better)",
+    sort_by: str = "Average (lower=better)",
     ascending: bool = True,
 ):
-    target_data = all_data[period]
-    target_metric = metric_to_sheet[metric]
-
-    if models:
-        target_model_size = [model_size_to_file_name[model] for model in models]
-        combined_data = pd.concat([target_data[model][target_metric] for model in target_model_size], axis=0)
-        combined_data["Name"] = combined_data["Name"].apply(lambda x: x.replace(".pth", ""))
-
-        # Filter models based on the size range
-        combined_data = combined_data[combined_data["Parameters Count (B)"].between(size_range[0], size_range[1])]
-
-        combined_data.reset_index(drop=True, inplace=True)
-
-        if "Average (The lower the better)" in combined_data.columns:
-            relevant_columns = [col for col in visible_columns if col not in ["Name", "Parameters Count (B)", "Average (The lower the better)"]]
-            if len(combined_data) > 0:
-                combined_data["Average (The lower the better)"] = round(combined_data[relevant_columns].mean(axis=1), 3)
-
-        if len(combined_data) > 0:
-            sorted_data = combined_data.sort_values(by=sort_by, ascending=ascending)
-            sorted_data = sorted_data.rename(columns={"Average (The lower the better)": "Average (lower=better)"})
-            sorted_data = sorted_data.rename(columns={"Parameters Count (B)": "Params (B)"})
-            visible_columns = ["Name", "Params (B)", "Average (lower=better)"] + visible_columns
-            filtered_data = sorted_data[visible_columns]
-
-            filtered_data.columns = [col.replace("_", " ") for col in filtered_data.columns]
-
-            formatter = {col: "{:.3f}" for col in filtered_data.columns if filtered_data[col].dtype in ["float64", "float32"]}
-
-            # color gradient
-            colors = ["#63be7b", "#ffffff", "#f8696b"]
-            cmap = LinearSegmentedColormap.from_list("custom_cmap", colors)
-            vmin = {}
-            vmax = {}
-            for column in filtered_data.columns:
-                if column in ["Name", "Params (B)"]:
-                    continue
-                col_values = filtered_data[column]
-                if len(col_values) > 1:
-                    second_largest = col_values.nlargest(2).iloc[-1]
-                    vmin[column] = col_values.min()
-                    vmax[column] = second_largest
-
-            target_color_columns = []
-            if "Average" in color_columns:
-                target_color_columns.append("Average (lower=better)")
-            if "Individual Tests" in color_columns:
-                target_color_columns.extend([col for col in filtered_data.columns if col not in ["Name", "Params (B)", "Average (lower=better)"]])
-
-            styler = filtered_data.style.format(formatter).applymap(color_cell, subset=["Params (B)"])
-
-            for column in target_color_columns:
-                if column in vmin and column in vmax:  # Ensure that the vmin and vmax dicts contain the column
-                    styler = styler.background_gradient(cmap=cmap, subset=[column], vmin=vmin[column], vmax=vmax[column])
-
-            return styler
-        else:
-            return pd.DataFrame()
-    else:
-        return pd.DataFrame()
+    print(
+        f"Updating - time: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, period: {period}, models: {models_size}, metric: {metric}, visible_columns: {visible_columns}, color_columns: {color_columns}, size_range: {size_range}, sort_by: {sort_by}, ascending: {ascending}\n"
+    )
+
+    if not models_size:
+        return "No data available for the selected models and period."
+        # return pd.DataFrame()
+
+    target_period_data = all_data[period]
+    target_file_name = [model_size_to_file_name[model] for model in models_size]
+    sheet_name = metric_to_sheet[metric]
+
+    # combined_data = pd.concat([target_period_data[file_name][sheet_name] for file_name in target_file_name], axis=0)
+    combined_data = pd.concat(
+        [df.dropna(axis=1, how="all") for df in [target_period_data[file_name][sheet_name] for file_name in target_file_name]], axis=0
+    )
+    if len(combined_data) == 0:
+        return "No data available for the selected models and period."
+        # return pd.DataFrame()
+
+    # Filter models based on the size range
+    combined_data = combined_data[combined_data["Parameters Count (B)"].between(size_range[0], size_range[1])]
+    combined_data.reset_index(drop=True, inplace=True)
+    if len(combined_data) == 0:
+        return "No data available for the selected models and period."
+        # return pd.DataFrame()
+
+    combined_data["Name"] = combined_data["Name"].apply(lambda x: x.replace(".pth", ""))
+
+    relevant_columns = [col for col in visible_columns if col not in ["Name", "Parameters Count (B)", "Average (The lower the better)"]]
+    if len(combined_data) > 0:
+        combined_data["Average (The lower the better)"] = round(combined_data[relevant_columns].mean(axis=1), 3)
+    combined_data = combined_data.rename(columns={"Parameters Count (B)": "Params (B)"})
+    combined_data = combined_data.rename(columns={"Average (The lower the better)": "Average (lower=better)"})
+    sorted_data = combined_data.sort_values(by=sort_by, ascending=ascending)
+    visible_columns = ["Name", "Params (B)", "Average (lower=better)"] + visible_columns
+    filtered_data = sorted_data[visible_columns]
+    filtered_data.columns = [col.replace("_", " ") for col in filtered_data.columns]
+
+    formatter = {col: "{:.3f}" for col in filtered_data.columns if filtered_data[col].dtype in ["float64", "float32"]}
+
+    # color gradient
+    colors = ["#63be7b", "#ffffff", "#f8696b"]
+    cmap = LinearSegmentedColormap.from_list("custom_cmap", colors)
+    vmin = {}
+    vmax = {}
+    for column in filtered_data.columns:
+        if column in ["Name", "Params (B)"]:
+            continue
+        col_values = filtered_data[column]
+        if len(col_values) > 1:
+            vmin[column] = col_values.min()
+            vmax[column] = col_values.max()
+
+    target_color_columns = []
+    if "Average" in color_columns:
+        target_color_columns.append("Average (lower=better)")
+    if "Individual Tests" in color_columns:
+        target_color_columns.extend([col for col in filtered_data.columns if col not in ["Name", "Params (B)", "Average (lower=better)"]])
+
+    styler = filtered_data.style.format(formatter).map(color_cell, subset=["Params (B)"])
+
+    for column in target_color_columns:
+        styler = styler.background_gradient(cmap=cmap, subset=[column], vmin=vmin[column], vmax=vmax[column])
+
+    # return styler
+
+    styler = styler.hide(axis="index")
+
+    widths = [300, 150, 150, 100, 100, 100, 100, 100, 100, 100, 100]
+    table_styles = []
+
+    for i, w in enumerate(widths):
+        table_styles.append(
+            {"selector": f"th.col{i}", "props": [("min-width", f"{w}px"), ("max-width", f"{w}px"), ("text-align", "center")]}  # 添加表头文字居中
+        )
+        table_styles.append(
+            {"selector": f"td.col{i}", "props": [("min-width", f"{w}px"), ("max-width", f"{w}px"), ("text-align", "center")]}  # 添加单元格文字居中
+        )
+
+    styler = styler.set_table_styles(table_styles)
+
+    html_output = styler.to_html()
+    return html_output
 
 
 def create_world_languages_gdp_chart():
@@ -292,95 +317,172 @@ def submit_model(name):
         return "ERROR: Unexpected error. Please try again later."
 
 
+# def create_scaling_plot(all_data, period):
+#     selected_columns = ["Name", "Parameters Count (B)", "Average (The lower the better)"]
+#     target_data = all_data[period]
+#     new_df = pd.DataFrame()
+
+#     for size in target_data.keys():
+#         new_df = pd.concat([new_df, target_data[size]["cr"].loc[:, selected_columns].dropna(axis=1, how="all")], axis=0)
+
+#     new_df.rename(columns={"Parameters Count (B)": "Params(B)", "Average (The lower the better)": "Compression Rate (%)"}, inplace=True)
+
+#     new_df["Log Params(B)"] = np.log(new_df["Params(B)"])
+#     new_df["Log Compression Rate (%)"] = np.log(new_df["Compression Rate (%)"])
+
+#     fig = px.scatter(
+#         new_df,
+#         x="Log Params(B)",
+#         y="Log Compression Rate (%)",
+#         title="Compression Rate Scaling Law",
+#         hover_name="Name",
+#         custom_data=["Params(B)", "Compression Rate (%)"],
+#     )
+
+#     fig.update_traces(
+#         hovertemplate="<b>%{hovertext}</b><br>Params(B): %{customdata[0]:.2f} B<br>Compression Rate (%): %{customdata[1]:.2f}<extra></extra>"
+#     )
+#     fig.update_layout(
+#         width=800,  # 设置图像宽度
+#         height=600,  # 设置图像高度
+#         title={"text": "Compression Rate Scaling Law", "x": 0.5, "xanchor": "center", "yanchor": "top"},
+#         showlegend=True,
+#         xaxis={"showgrid": True, "zeroline": False, "type": "linear", "title": "Params(B)"},  # 确保坐标轴类型正确
+#         yaxis={"showgrid": True, "zeroline": False, "type": "linear", "title": "Compression Rate (%)", "autorange": "reversed"},
+#     )
+
+#     names_to_connect_dict = {
+#         "2024-05": ["Meta-Llama-3-8B", "stablelm-3b-4e1t", "Qwen2-1.5B", "TinyLlama-1.1B-intermediate-step-1431k-3T", "Mistral-Nemo-Base-2407"],
+#         "2024-06": ["Meta-Llama-3-8B", "stablelm-3b-4e1t", "Qwen2-1.5B", "TinyLlama-1.1B-intermediate-step-1431k-3T", "Mistral-Nemo-Base-2407"],
+#         "2024-07": ["Meta-Llama-3.1-8B", "stablelm-3b-4e1t", "Qwen2-1.5B", "TinyLlama-1.1B-intermediate-step-1431k-3T", "Mistral-Nemo-Base-2407"],
+#         "2024-08": [
+#             "Meta-Llama-3.1-8B",
+#             "Rene-v0.1-1.3b-pytorch",
+#             "stablelm-3b-4e1t",
+#             "Qwen2-1.5B",
+#             "TinyLlama-1.1B-intermediate-step-1431k-3T",
+#             "Mistral-Nemo-Base-2407",
+#         ],
+#         "2025-01": ["Qwen2.5-1.5B"],
+#     }
+
+#     names_to_connect = names_to_connect_dict.get(period, names_to_connect_dict["2024-08"])
+
+#     connection_points = new_df[new_df["Name"].isin(names_to_connect)]
+#     print(connection_points)
+
+#     new_df["Color"] = new_df["Name"].apply(lambda name: "#39C5BB" if name in names_to_connect else "#636efa")
+
+#     fig.update_traces(marker=dict(color=new_df["Color"]))
+
+#     X = connection_points["Log Params(B)"].values.reshape(-1, 1)
+#     y = connection_points["Log Compression Rate (%)"].values
+#     model = LinearRegression().fit(X, y)
+
+#     x_min = connection_points["Log Params(B)"].min()
+#     x_max = connection_points["Log Params(B)"].max()
+#     extended_x = np.linspace(x_min, x_max * 1.5, 100)
+#     extended_x_original = np.exp(extended_x)
+#     trend_line_y = model.predict(extended_x.reshape(-1, 1))
+#     trend_line_y_original = np.exp(trend_line_y)
+
+#     trend_line = go.Scatter(
+#         x=extended_x,
+#         y=trend_line_y,
+#         mode="lines",
+#         line=dict(color="skyblue", dash="dash"),
+#         name="Trend Line",
+#         hovertemplate="<b>Params(B):</b> %{customdata[0]:.2f}<br>" + "<b>Compression Rate (%):</b> %{customdata[1]:.2f}<extra></extra>",
+#         customdata=np.stack((extended_x_original, trend_line_y_original), axis=-1),
+#     )
+
+#     fig.add_trace(trend_line)
+
+#     x_min = new_df["Params(B)"].min()
+#     x_max = new_df["Params(B)"].max()
+#     x_tick_vals = np.geomspace(x_min, x_max, num=5)
+#     x_tick_text = [f"{val:.1f}" for val in x_tick_vals]
+
+#     y_min = new_df["Compression Rate (%)"].min()
+#     y_max = new_df["Compression Rate (%)"].max()
+#     y_tick_vals = np.geomspace(y_min, y_max, num=5)
+#     y_tick_text = [f"{val:.1f}" for val in y_tick_vals]
+
+#     fig.update_xaxes(tickvals=np.log(x_tick_vals), ticktext=x_tick_text, title="Params(B)")
+#     fig.update_yaxes(tickvals=np.log(y_tick_vals), ticktext=y_tick_text, title="Compression Rate (%)", autorange="reversed")
+
+#     fig.update_layout(xaxis=dict(showgrid=True, zeroline=False), yaxis=dict(showgrid=True, zeroline=False))
+
+#     fig.update_traces(marker=dict(size=12))
+
+#     print(fig.layout)
+
+#     return fig
+
+
 def create_scaling_plot(all_data, period):
     selected_columns = ["Name", "Parameters Count (B)", "Average (The lower the better)"]
     target_data = all_data[period]
     new_df = pd.DataFrame()
 
     for size in target_data.keys():
-        new_df = pd.concat([new_df, target_data[size]["cr"].loc[:, selected_columns]], axis=0)
-
-    new_df.rename(columns={"Parameters Count (B)": "Params(B)", "Average (The lower the better)": "Compression Rate (%)"}, inplace=True)
-
-    new_df["Log Params(B)"] = np.log(new_df["Params(B)"])
-    new_df["Log Compression Rate (%)"] = np.log(new_df["Compression Rate (%)"])
-
-    fig = px.scatter(
-        new_df,
-        x="Log Params(B)",
-        y="Log Compression Rate (%)",
-        title="Compression Rate Scaling Law",
-        hover_name="Name",
-        custom_data=["Params(B)", "Compression Rate (%)"],
-    )
-
-    fig.update_traces(
-        hovertemplate="<b>%{hovertext}</b><br>Params(B): %{customdata[0]:.2f} B<br>Compression Rate (%): %{customdata[1]:.2f}<extra></extra>"
+        new_df = pd.concat([new_df, target_data[size]["cr"].loc[:, selected_columns].dropna(axis=1, how="all")], axis=0)
+
+    x_values = new_df["Parameters Count (B)"].astype(float).tolist()
+    y_values = new_df["Average (The lower the better)"].astype(float).tolist()
+    names = new_df["Name"].tolist()
+
+    # 计算对数空间的范围
+    x_min, x_max = np.log10(min(x_values)), np.log10(max(x_values))
+    y_min, y_max = np.log10(min(y_values)), np.log10(max(y_values))
+
+    # 计算合适的刻度间隔
+    x_dtick = (x_max - x_min) / 4  # 分成5个刻度
+    y_dtick = (y_max - y_min) / 4
+
+    fig = go.Figure()
+
+    fig.add_trace(
+        go.Scatter(
+            x=x_values,
+            y=y_values,
+            mode="markers",
+            name="Models",
+            marker=dict(size=12, color="#39C5BB", opacity=0.8),
+            text=names,
+            customdata=list(zip(x_values, y_values)),
+            hovertemplate=(
+                "<b>%{text}</b><br>" + "Params: %{customdata[0]:.2f}B<br>" + "Compression Rate: %{customdata[1]:.2f}%<br>" + "<extra></extra>"
+            ),
+        )
     )
 
-    names_to_connect_dict = {
-        "2024-05": ["Meta-Llama-3-8B", "stablelm-3b-4e1t", "Qwen2-1.5B", "TinyLlama-1.1B-intermediate-step-1431k-3T", "Mistral-Nemo-Base-2407"],
-        "2024-06": ["Meta-Llama-3-8B", "stablelm-3b-4e1t", "Qwen2-1.5B", "TinyLlama-1.1B-intermediate-step-1431k-3T", "Mistral-Nemo-Base-2407"],
-        "2024-07": ["Meta-Llama-3.1-8B", "stablelm-3b-4e1t", "Qwen2-1.5B", "TinyLlama-1.1B-intermediate-step-1431k-3T", "Mistral-Nemo-Base-2407"],
-        "2024-08": [
-            "Meta-Llama-3.1-8B",
-            "Rene-v0.1-1.3b-pytorch",
-            "stablelm-3b-4e1t",
-            "Qwen2-1.5B",
-            "TinyLlama-1.1B-intermediate-step-1431k-3T",
-            "Mistral-Nemo-Base-2407",
-        ],
-        "2025-01": ["Qwen2.5-1.5B"],
-    }
-
-    names_to_connect = names_to_connect_dict.get(period, names_to_connect_dict["2024-08"])
-
-    connection_points = new_df[new_df["Name"].isin(names_to_connect)]
-
-    new_df["Color"] = new_df["Name"].apply(lambda name: "#39C5BB" if name in names_to_connect else "#636efa")
-
-    fig.update_traces(marker=dict(color=new_df["Color"]))
-
-    X = connection_points["Log Params(B)"].values.reshape(-1, 1)
-    y = connection_points["Log Compression Rate (%)"].values
-    model = LinearRegression().fit(X, y)
-
-    x_min = connection_points["Log Params(B)"].min()
-    x_max = connection_points["Log Params(B)"].max()
-    extended_x = np.linspace(x_min, x_max * 1.5, 100)
-    extended_x_original = np.exp(extended_x)
-    trend_line_y = model.predict(extended_x.reshape(-1, 1))
-    trend_line_y_original = np.exp(trend_line_y)
-
-    trend_line = go.Scatter(
-        x=extended_x,
-        y=trend_line_y,
-        mode="lines",
-        line=dict(color="skyblue", dash="dash"),
-        name="Trend Line",
-        hovertemplate="<b>Params(B):</b> %{customdata[0]:.2f}<br>" + "<b>Compression Rate (%):</b> %{customdata[1]:.2f}<extra></extra>",
-        customdata=np.stack((extended_x_original, trend_line_y_original), axis=-1),
+    fig.update_layout(
+        title={"text": "Compression Rate Scaling Law", "x": 0.5, "xanchor": "center", "yanchor": "top"},
+        width=800,
+        height=600,
+        showlegend=True,
+        xaxis=dict(
+            title="Parameters (B)",
+            showgrid=True,
+            zeroline=False,
+            type="log",
+            dtick=x_dtick,
+            tickformat=".2f",  # 保留两位小数
+            range=[x_min - 0.1, x_max + 0.1],
+        ),
+        yaxis=dict(
+            title="Compression Rate (%)",
+            showgrid=True,
+            zeroline=False,
+            type="log",
+            dtick=y_dtick,
+            tickformat=".2f",  # 保留两位小数
+            range=[y_min - 0.1, y_max + 0.1],
+            autorange="reversed",
+        ),
     )
 
-    fig.add_trace(trend_line)
-
-    x_min = new_df["Params(B)"].min()
-    x_max = new_df["Params(B)"].max()
-    x_tick_vals = np.geomspace(x_min, x_max, num=5)
-    x_tick_text = [f"{val:.1f}" for val in x_tick_vals]
-
-    y_min = new_df["Compression Rate (%)"].min()
-    y_max = new_df["Compression Rate (%)"].max()
-    y_tick_vals = np.geomspace(y_min, y_max, num=5)
-    y_tick_text = [f"{val:.1f}" for val in y_tick_vals]
-
-    fig.update_xaxes(tickvals=np.log(x_tick_vals), ticktext=x_tick_text, title="Params(B)")
-    fig.update_yaxes(tickvals=np.log(y_tick_vals), ticktext=y_tick_text, title="Compression Rate (%)", autorange="reversed")
-
-    fig.update_layout(xaxis=dict(showgrid=True, zeroline=False), yaxis=dict(showgrid=True, zeroline=False))
-
-    fig.update_traces(marker=dict(size=12))
-
     return fig
 
 
@@ -423,19 +525,17 @@ time_list.sort()
 last_period = time_list[-1]
 
 initial_fig = create_scaling_plot(all_data, last_period)
-initial_period = last_period
-initial_models = model_size_list
 initial_metric = metric_list[0]
 initial_columns = get_unique_column_names(all_data)
-initial_columns = initial_columns[:-1]
-# initial_colors = ["Average"]
+# initial_columns = initial_columns[:-1]
 initial_colors = ["Average", "Individual Tests"]
 initial_size_range = [0, 15]
-initial_data = update_table(initial_period, initial_models, initial_metric, initial_columns, initial_colors, initial_size_range)
+initial_data = update_table(last_period, model_size_list, initial_metric, initial_columns, initial_colors, initial_size_range)
 
 css = """
 .gradio-container {
     max-width: 95% !important;
+    margin: 0 auto;
 }
 .tab-buttons button {
     font-size: 1.3em;
@@ -444,7 +544,11 @@ css = """
     white-space: normal;
     word-break: break-word;
 }
-
+table {
+    margin-left: auto !important;
+    margin-right: auto !important;
+    width: 100% !important;
+}
 """
 
 TITLE_HTML = '<h1 style="text-align:center"><span style="font-size:1.3em">🏆 LLM Compression Leaderboard</span></h1>'
@@ -460,17 +564,18 @@ with gr.Blocks(css=css) as demo:
                     period_selector = gr.Dropdown(label="Period", choices=time_list, value=last_period)
                     model_selector = gr.CheckboxGroup(label="Model Size", choices=model_size_list, value=model_size_list)
                     size_range_slider = RangeSlider(minimum=0, maximum=15, value=[0, 15], step=0.1, label="Model Size Range")
-                    metric_selector = gr.Dropdown(label="Metric", choices=metric_list, value=metric_list[0])
+                    metric_selector = gr.Dropdown(label="Metric", choices=metric_list, value=initial_metric)
                 with gr.Column():
                     color_selector = gr.CheckboxGroup(label="Colored Columns", choices=["Average", "Individual Tests"], value=initial_colors)
                     colfilter = gr.CheckboxGroup(label="Data Source", choices=get_unique_column_names(all_data), value=initial_columns)
 
-            table = gr.Dataframe(
-                initial_data,
-                column_widths=[130, 50, 50, 35, 35, 35, 35, 35, 35, 35, 35],
-                wrap=True,
-                height=800,
-            )
+            # table = gr.Dataframe(
+            #     initial_data,
+            #     column_widths=[130, 50, 50, 35, 35, 35, 35, 35, 35, 35, 35],
+            #     wrap=True,
+            #     max_height=800,
+            # )
+            table = gr.HTML(initial_data)
 
             period_selector.change(
                 update_table, inputs=[period_selector, model_selector, metric_selector, colfilter, color_selector, size_range_slider], outputs=table
@@ -494,6 +599,7 @@ with gr.Blocks(css=css) as demo:
         with gr.Tab("🌍 MultiLang"):
             gr.Markdown("## Coming soon...")
             world_languages_plot = gr.Plot(create_world_languages_gdp_chart())
+
         with gr.Tab("📈 Scaling Law"):
             print(time_list)
             period_selector_2 = gr.Dropdown(label="Period", choices=time_list, value=last_period)
@@ -507,13 +613,13 @@ with gr.Blocks(css=css) as demo:
 
         with gr.Tab("ℹ️ About"):
             gr.Markdown(about_md)
+
         with gr.Tab("🚀 Submit"):
             with gr.Group():
                 with gr.Row():
                     model_name = gr.Textbox(max_lines=1, placeholder="Enter model name...", show_label=False, scale=4)
                     submit = gr.Button("Submit", variant="primary", scale=0)
             output = gr.Markdown("# Enter a public HF repo id, then hit Submit to add it to the evaluation queue.")
-
             submit.click(fn=submit_model, inputs=model_name, outputs=output)
 
 demo.launch(share=False)

requirements.txt

@@ -1,6 +1,6 @@
 plotly==5.22.0
 scikit-learn==1.5.0
-gradio==4.31.5
+gradio==5.15.0
 huggingface_hub==0.23.1
 pandas==2.2.2
 openpyxl==3.1.2