Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import os | |
| import zipfile | |
| import base64 | |
| CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" | |
| CITATION_BUTTON_TEXT = r"""@misc{aienergyscore-leaderboard, | |
| author = {Sasha Luccioni and Boris Gamazaychikov and Emma Strubell and Sara Hooker and Yacine Jernite and Carole-Jean Wu and Margaret Mitchell}, | |
| title = {AI Energy Score Leaderboard - February 2025}, | |
| year = {2025}, | |
| publisher = {Hugging Face}, | |
| howpublished = "\url{https://huggingface.co/spaces/AIEnergyScore/Leaderboard}", | |
| }""" | |
| # List of tasks (CSV filenames) | |
| tasks = [ | |
| 'asr.csv', | |
| 'object_detection.csv', | |
| 'text_classification.csv', | |
| 'image_captioning.csv', | |
| 'question_answering.csv', | |
| 'text_generation.csv', | |
| 'image_classification.csv', | |
| 'sentence_similarity.csv', | |
| 'image_generation.csv', | |
| 'summarization.csv' | |
| ] | |
| def format_stars(score): | |
| try: | |
| score_int = int(score) | |
| except Exception: | |
| score_int = 0 | |
| # Render stars in green with a slightly larger font. | |
| return f'<span style="color: #3fa45bff; font-size:1.5em;">{"★" * score_int}</span>' | |
| def make_link(mname): | |
| parts = str(mname).split('/') | |
| display_name = parts[1] if len(parts) > 1 else mname | |
| return f'<a href="https://huggingface.co/{mname}" target="_blank">{display_name}</a>' | |
| def extract_link_text(html_link): | |
| """Extracts the inner text from an HTML link.""" | |
| start = html_link.find('>') + 1 | |
| end = html_link.rfind('</a>') | |
| if start > 0 and end > start: | |
| return html_link[start:end] | |
| else: | |
| return html_link | |
| def generate_html_table_from_df(df): | |
| """ | |
| Generates an HTML table with four columns: | |
| - Model (with link) | |
| - Provider (extracted from the model field) | |
| - GPU Energy (Wh) plus a horizontal bar | |
| - Score (as stars) | |
| """ | |
| if not df.empty: | |
| max_length = max(len(extract_link_text(link)) for link in df['Model']) | |
| else: | |
| max_length = 10 | |
| static_width = max_length * 10 + 16 | |
| max_energy = df['gpu_energy_numeric'].max() if not df.empty else 1 | |
| color_map = {"1": "black", "2": "black", "3": "black", "4": "black", "5": "black"} | |
| html = '<table style="width:100%; border-collapse: collapse; font-family: Inter, sans-serif;">' | |
| html += '<thead><tr style="background-color: #f2f2f2;">' | |
| html += '<th style="text-align: left; padding: 8px;" title="Model name with link to Hugging Face">Model</th>' | |
| html += '<th style="text-align: left; padding: 8px;" title="AI Provider extracted from the model name">Provider</th>' | |
| html += '<th style="text-align: left; padding: 8px;" title="GPU energy consumed in Watt-hours for 1,000 queries">GPU Energy (Wh)</th>' | |
| html += '<th style="text-align: left; padding: 8px;" title="Energy efficiency score">Score</th>' | |
| html += '</tr></thead>' | |
| html += '<tbody>' | |
| for _, row in df.iterrows(): | |
| energy_numeric = row['gpu_energy_numeric'] | |
| energy_str = f"{energy_numeric:.2f}" | |
| bar_width = (energy_numeric / max_energy) * 100 | |
| score_val = row['energy_score'] | |
| bar_color = color_map.get(str(score_val), "gray") | |
| html += '<tr>' | |
| html += f'<td style="padding: 8px; width: {static_width}px;">{row["Model"]}</td>' | |
| html += f'<td style="padding: 8px;">{row["Provider"]}</td>' | |
| html += ( | |
| f'<td style="padding: 8px;">{energy_str}<br>' | |
| f'<div style="background-color: {bar_color}; width: {bar_width:.1f}%; height: 10px;"></div></td>' | |
| ) | |
| html += f'<td style="padding: 8px;">{row["Score"]}</td>' | |
| html += '</tr>' | |
| html += '</tbody></table>' | |
| return f'<div class="table-container">{html}</div>' | |
| # --- Functions for creating the efficiency difference callout cards --- | |
| def get_efficiency_diff_for_all(): | |
| """Calculates the efficiency difference across all models.""" | |
| all_df = pd.DataFrame() | |
| for task in tasks: | |
| df = pd.read_csv('data/energy/' + task) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| all_df = pd.concat([all_df, df], ignore_index=True) | |
| if all_df.empty: | |
| return "<div>No data available</div>" | |
| min_val = all_df['gpu_energy_numeric'].min() | |
| max_val = all_df['gpu_energy_numeric'].max() | |
| diff = max_val - min_val | |
| # A colorful gradient card for global stats. | |
| return ( | |
| f"<div style='background: linear-gradient(135deg, #f6d365, #fda085); padding: 15px; " | |
| f"border-radius: 8px; margin: 10px; color: #333;'>" | |
| f"<strong>All Models:</strong> Efficiency difference is <strong>{diff:.2f} Wh</strong> " | |
| f"(min: {min_val:.2f} Wh, max: {max_val:.2f} Wh)" | |
| f"</div>" | |
| ) | |
| def get_efficiency_diff_for_task(task_filename): | |
| """Calculates the efficiency difference for models in a given task.""" | |
| df = pd.read_csv('data/energy/' + task_filename) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| if df.empty: | |
| return "<div>No data available</div>" | |
| min_val = df['gpu_energy_numeric'].min() | |
| max_val = df['gpu_energy_numeric'].max() | |
| diff = max_val - min_val | |
| # A different gradient for the selected task | |
| return ( | |
| f"<div style='background: linear-gradient(135deg, #a8e063, #56ab2f); padding: 15px; " | |
| f"border-radius: 8px; margin: 10px; color: #333;'>" | |
| f"<strong>Selected Task:</strong> Efficiency difference is <strong>{diff:.2f} Wh</strong> " | |
| f"(min: {min_val:.2f} Wh, max: {max_val:.2f} Wh)" | |
| f"</div>" | |
| ) | |
| # --- Function to zip all CSV files (unchanged) --- | |
| def zip_csv_files(): | |
| data_dir = "data/energy" | |
| zip_filename = "data.zip" | |
| with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: | |
| for filename in os.listdir(data_dir): | |
| if filename.endswith(".csv"): | |
| filepath = os.path.join(data_dir, filename) | |
| zipf.write(filepath, arcname=filename) | |
| return zip_filename | |
| def get_zip_data_link(): | |
| zip_filename = zip_csv_files() | |
| with open(zip_filename, "rb") as f: | |
| data = f.read() | |
| b64 = base64.b64encode(data).decode() | |
| href = ( | |
| f'<a href="data:application/zip;base64,{b64}" ' | |
| 'download="data.zip" ' | |
| 'style="text-decoration: none; font-weight: bold; font-size: 1.1em; ' | |
| 'color: black; font-family: \'Inter\', sans-serif;">Download Data</a>' | |
| ) | |
| return href | |
| # --- Modified functions to include a sort_order parameter --- | |
| def get_model_names_html(task, sort_order="Low to High"): | |
| df = pd.read_csv('data/energy/' + task) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['energy_score'] = df['energy_score'].astype(int) | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| # Add Provider column (text before the slash in the model field) | |
| df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
| df['Model'] = df['model'].apply(make_link) | |
| df['Score'] = df['energy_score'].apply(format_stars) | |
| ascending = (sort_order == "Low to High") | |
| df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
| return generate_html_table_from_df(df) | |
| def get_all_model_names_html(sort_order="Low to High"): | |
| all_df = pd.DataFrame() | |
| for task in tasks: | |
| df = pd.read_csv('data/energy/' + task) | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| df['energy_score'] = df['energy_score'].astype(int) | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
| df['Model'] = df['model'].apply(make_link) | |
| df['Score'] = df['energy_score'].apply(format_stars) | |
| all_df = pd.concat([all_df, df], ignore_index=True) | |
| all_df = all_df.drop_duplicates(subset=['model']) | |
| ascending = (sort_order == "Low to High") | |
| all_df = all_df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
| return generate_html_table_from_df(all_df) | |
| def get_text_generation_model_names_html(model_class, sort_order="Low to High"): | |
| df = pd.read_csv('data/energy/text_generation.csv') | |
| if df.columns[0].startswith("Unnamed:"): | |
| df = df.iloc[:, 1:] | |
| if 'class' in df.columns: | |
| df = df[df['class'] == model_class] | |
| df['energy_score'] = df['energy_score'].astype(int) | |
| df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 | |
| df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) | |
| df['Model'] = df['model'].apply(make_link) | |
| df['Score'] = df['energy_score'].apply(format_stars) | |
| ascending = (sort_order == "Low to High") | |
| df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) | |
| return generate_html_table_from_df(df) | |
| # --- Update functions for dropdown changes --- | |
| def update_text_generation(selected_display, sort_order): | |
| mapping = { | |
| "A (Single Consumer GPU) <20B parameters": "A", | |
| "B (Single Cloud GPU) 20-66B parameters": "B", | |
| "C (Multiple Cloud GPUs) >66B parameters": "C" | |
| } | |
| model_class = mapping.get(selected_display, "A") | |
| table_html = get_text_generation_model_names_html(model_class, sort_order) | |
| # Update the task-specific callout for text generation | |
| task_diff_html = get_efficiency_diff_for_task('text_generation.csv') | |
| return table_html, task_diff_html | |
| def update_image_generation(sort_order): | |
| table_html = get_model_names_html('image_generation.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('image_generation.csv') | |
| return table_html, task_diff_html | |
| def update_text_classification(sort_order): | |
| table_html = get_model_names_html('text_classification.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('text_classification.csv') | |
| return table_html, task_diff_html | |
| def update_image_classification(sort_order): | |
| table_html = get_model_names_html('image_classification.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('image_classification.csv') | |
| return table_html, task_diff_html | |
| def update_image_captioning(sort_order): | |
| table_html = get_model_names_html('image_captioning.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('image_captioning.csv') | |
| return table_html, task_diff_html | |
| def update_summarization(sort_order): | |
| table_html = get_model_names_html('summarization.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('summarization.csv') | |
| return table_html, task_diff_html | |
| def update_asr(sort_order): | |
| table_html = get_model_names_html('asr.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('asr.csv') | |
| return table_html, task_diff_html | |
| def update_object_detection(sort_order): | |
| table_html = get_model_names_html('object_detection.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('object_detection.csv') | |
| return table_html, task_diff_html | |
| def update_sentence_similarity(sort_order): | |
| table_html = get_model_names_html('sentence_similarity.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('sentence_similarity.csv') | |
| return table_html, task_diff_html | |
| def update_extractive_qa(sort_order): | |
| table_html = get_model_names_html('question_answering.csv', sort_order) | |
| task_diff_html = get_efficiency_diff_for_task('question_answering.csv') | |
| return table_html, task_diff_html | |
| def update_all_tasks(sort_order): | |
| return get_all_model_names_html(sort_order) | |
| # --- Build the Gradio Interface --- | |
| demo = gr.Blocks(css=""" | |
| .gr-dataframe table { | |
| table-layout: fixed; | |
| width: 100%; | |
| } | |
| .gr-dataframe th, .gr-dataframe td { | |
| max-width: 150px; | |
| white-space: nowrap; | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| } | |
| .table-container { | |
| width: 100%; | |
| margin-left: auto; | |
| margin-right: auto; | |
| } | |
| """) | |
| with demo: | |
| # --- Header Links --- | |
| gr.HTML(f''' | |
| <div style="display: flex; justify-content: space-evenly; align-items: center; margin-bottom: 20px;"> | |
| <a href="https://huggingface.co/spaces/AIEnergyScore/submission_portal" style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: black; font-family: 'Inter', sans-serif;">Submission Portal</a> | |
| <a href="https://huggingface.co/spaces/AIEnergyScore/Label" style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: black; font-family: 'Inter', sans-serif;">Label Generator</a> | |
| <a href="https://huggingface.github.io/AIEnergyScore/#faq" style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: black; font-family: 'Inter', sans-serif;">FAQ</a> | |
| <a href="https://huggingface.github.io/AIEnergyScore/#documentation" style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: black; font-family: 'Inter', sans-serif;">Documentation</a> | |
| {get_zip_data_link()} | |
| <a href="https://huggingface.co/spaces/AIEnergyScore/README/discussions" style="text-decoration: none; font-weight: bold; font-size: 1.1em; color: black; font-family: 'Inter', sans-serif;">Community</a> | |
| </div> | |
| ''') | |
| # --- Logo and Subtitle --- | |
| gr.HTML(''' | |
| <div style="margin-top: 0px; text-align: center;"> | |
| <img src="https://huggingface.co/spaces/AIEnergyScore/Leaderboard/resolve/main/logo.png" | |
| alt="Logo" | |
| style="max-width: 300px; height: auto; margin-bottom: 10px;"> | |
| </div> | |
| ''') | |
| gr.Markdown('<div style="text-align: center; font-size: 1.2em;">Welcome to the AI Energy Score leaderboard. Select different tasks to see scored models.</div>') | |
| # --- Callout Cards (Row at the Top) --- | |
| with gr.Row(): | |
| all_models_card = gr.HTML(get_efficiency_diff_for_all()) | |
| # Initially, we show the stats for text_generation as default for the selected task. | |
| selected_task_card = gr.HTML(get_efficiency_diff_for_task('text_generation.csv')) | |
| # --- Tabs for the Different Tasks --- | |
| with gr.Tabs(): | |
| # --- Text Generation Tab --- | |
| with gr.TabItem("Text Generation 💬"): | |
| with gr.Row(): | |
| model_class_options = [ | |
| "A (Single Consumer GPU) <20B parameters", | |
| "B (Single Cloud GPU) 20-66B parameters", | |
| "C (Multiple Cloud GPUs) >66B parameters" | |
| ] | |
| model_class_dropdown = gr.Dropdown( | |
| choices=model_class_options, | |
| label="Select Model Class", | |
| value=model_class_options[0] | |
| ) | |
| sort_dropdown_tg = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| # Two outputs: the table and the task callout card. | |
| tg_table = gr.HTML(get_text_generation_model_names_html("A", "Low to High")) | |
| model_class_dropdown.change( | |
| fn=update_text_generation, | |
| inputs=[model_class_dropdown, sort_dropdown_tg], | |
| outputs=[tg_table, selected_task_card] | |
| ) | |
| sort_dropdown_tg.change( | |
| fn=update_text_generation, | |
| inputs=[model_class_dropdown, sort_dropdown_tg], | |
| outputs=[tg_table, selected_task_card] | |
| ) | |
| # --- Image Generation Tab --- | |
| with gr.TabItem("Image Generation 📷"): | |
| sort_dropdown_img = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| img_table = gr.HTML(get_model_names_html('image_generation.csv', "Low to High")) | |
| sort_dropdown_img.change( | |
| fn=update_image_generation, | |
| inputs=sort_dropdown_img, | |
| outputs=[img_table, selected_task_card] | |
| ) | |
| # --- Text Classification Tab --- | |
| with gr.TabItem("Text Classification 🎭"): | |
| sort_dropdown_tc = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| tc_table = gr.HTML(get_model_names_html('text_classification.csv', "Low to High")) | |
| sort_dropdown_tc.change( | |
| fn=update_text_classification, | |
| inputs=sort_dropdown_tc, | |
| outputs=[tc_table, selected_task_card] | |
| ) | |
| # --- Image Classification Tab --- | |
| with gr.TabItem("Image Classification 🖼️"): | |
| sort_dropdown_ic = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| ic_table = gr.HTML(get_model_names_html('image_classification.csv', "Low to High")) | |
| sort_dropdown_ic.change( | |
| fn=update_image_classification, | |
| inputs=sort_dropdown_ic, | |
| outputs=[ic_table, selected_task_card] | |
| ) | |
| # --- Image Captioning Tab --- | |
| with gr.TabItem("Image Captioning 📝"): | |
| sort_dropdown_icap = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| icap_table = gr.HTML(get_model_names_html('image_captioning.csv', "Low to High")) | |
| sort_dropdown_icap.change( | |
| fn=update_image_captioning, | |
| inputs=sort_dropdown_icap, | |
| outputs=[icap_table, selected_task_card] | |
| ) | |
| # --- Summarization Tab --- | |
| with gr.TabItem("Summarization 📃"): | |
| sort_dropdown_sum = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| sum_table = gr.HTML(get_model_names_html('summarization.csv', "Low to High")) | |
| sort_dropdown_sum.change( | |
| fn=update_summarization, | |
| inputs=sort_dropdown_sum, | |
| outputs=[sum_table, selected_task_card] | |
| ) | |
| # --- Automatic Speech Recognition Tab --- | |
| with gr.TabItem("Automatic Speech Recognition 💬"): | |
| sort_dropdown_asr = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| asr_table = gr.HTML(get_model_names_html('asr.csv', "Low to High")) | |
| sort_dropdown_asr.change( | |
| fn=update_asr, | |
| inputs=sort_dropdown_asr, | |
| outputs=[asr_table, selected_task_card] | |
| ) | |
| # --- Object Detection Tab --- | |
| with gr.TabItem("Object Detection 🚘"): | |
| sort_dropdown_od = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| od_table = gr.HTML(get_model_names_html('object_detection.csv', "Low to High")) | |
| sort_dropdown_od.change( | |
| fn=update_object_detection, | |
| inputs=sort_dropdown_od, | |
| outputs=[od_table, selected_task_card] | |
| ) | |
| # --- Sentence Similarity Tab --- | |
| with gr.TabItem("Sentence Similarity 📚"): | |
| sort_dropdown_ss = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| ss_table = gr.HTML(get_model_names_html('sentence_similarity.csv', "Low to High")) | |
| sort_dropdown_ss.change( | |
| fn=update_sentence_similarity, | |
| inputs=sort_dropdown_ss, | |
| outputs=[ss_table, selected_task_card] | |
| ) | |
| # --- Extractive QA Tab --- | |
| with gr.TabItem("Extractive QA ❔"): | |
| sort_dropdown_qa = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| qa_table = gr.HTML(get_model_names_html('question_answering.csv', "Low to High")) | |
| sort_dropdown_qa.change( | |
| fn=update_extractive_qa, | |
| inputs=sort_dropdown_qa, | |
| outputs=[qa_table, selected_task_card] | |
| ) | |
| # --- All Tasks Tab (only table update) --- | |
| with gr.TabItem("All Tasks 💡"): | |
| sort_dropdown_all = gr.Dropdown( | |
| choices=["Low to High", "High to Low"], | |
| label="Sort", | |
| value="Low to High" | |
| ) | |
| all_table = gr.HTML(get_all_model_names_html("Low to High")) | |
| sort_dropdown_all.change(fn=update_all_tasks, inputs=sort_dropdown_all, outputs=all_table) | |
| with gr.Accordion("📙 Citation", open=False): | |
| citation_button = gr.Textbox( | |
| value=CITATION_BUTTON_TEXT, | |
| label=CITATION_BUTTON_LABEL, | |
| elem_id="citation-button", | |
| lines=10, | |
| show_copy_button=True, | |
| ) | |
| gr.Markdown("Last updated: February 2025") | |
| demo.launch() |