Final Changes

app.py

@@ -4,14 +4,16 @@ import os
 from gradio_rangeslider import RangeSlider
 
 from src.filter_utils import filter, filter_cols
+from src.process_data import merge_data
+import assets.text_content as tc
 
 # Main Leaderboard containing everything
 text_leaderboard = pd.read_csv(os.path.join('assets', 'merged_data.csv'))
-text_leaderboard = text_leaderboard.sort_values(by='Clemscore', ascending=False)  
+text_leaderboard = text_leaderboard.sort_values(by=tc.CLEMSCORE, ascending=False)  
 
-open_weight_df = text_leaderboard[text_leaderboard['Open Weight'] == True]
+open_weight_df = text_leaderboard[text_leaderboard[tc.OPEN_WEIGHT] == True]
 if not open_weight_df.empty:  # Check if filtered df is non-empty
-    max_parameter_size = open_weight_df['Parameters (B)'].max()
+    max_parameter_size = open_weight_df[tc.PARAMS].max()
 
 # Short leaderboard containing fixed columns
 short_leaderboard = filter_cols(text_leaderboard)
@@ -27,18 +29,18 @@ contexts = []
 dates = []
 
 for i in range(len(text_leaderboard)):
-    lang_splits = text_leaderboard.iloc[i]['Languages'].split(',')
+    lang_splits = text_leaderboard.iloc[i][tc.LANGS].split(',')
     lang_splits = [s.strip() for s in lang_splits]
     langs += lang_splits
-    license_name = text_leaderboard.iloc[i]['License Name']
+    license_name = text_leaderboard.iloc[i][tc.LICENSE_NAME]
 
     licenses.append(license_name)
-    ip_prices.append(text_leaderboard.iloc[i]['Input $/1M tokens'])
-    op_prices.append(text_leaderboard.iloc[i]['Output $/1M tokens'])
-    latencies.append(text_leaderboard.iloc[i]['Latency (s)'])
-    parameters.append(text_leaderboard.iloc[i]['Parameters (B)'])
-    contexts.append(text_leaderboard.iloc[i]['Context Size (k)'])
-    dates.append(text_leaderboard.iloc[i]['Release Date'])
+    ip_prices.append(text_leaderboard.iloc[i][tc.INPUT])
+    op_prices.append(text_leaderboard.iloc[i][tc.OUTPUT])
+    latencies.append(text_leaderboard.iloc[i][tc.LATENCY])
+    parameters.append(text_leaderboard.iloc[i][tc.PARAMS])
+    contexts.append(text_leaderboard.iloc[i][tc.CONTEXT])
+    dates.append(text_leaderboard.iloc[i][tc.RELEASE_DATE])
 
 
 langs = list(set(langs))
@@ -54,7 +56,6 @@ max_latency = max(latencies)
 min_parameters = 0 if pd.isna(min(parameters)) else min(parameters)
 max_parameter = max_parameter_size
 parameter_step = 1
-print(f"MIN {min_parameters}, MAX {max_parameter}")
 
 min_context = min(contexts)
 max_context = max(contexts)
@@ -63,21 +64,14 @@ context_step = 8
 min_date = min(dates)
 max_date = max(dates)
 
-TITLE = """<h1 align="center" id="space-title"> LLM Calculator ⚖️⚡ 📏💰</h1>"""
-CSS = """
-#double-slider-1 {height: 100px}
-#double-slider-2 {height: 100px}
-#double-slider-3 {height: 100px}
-#double-slider-4 {height: 100px}
-"""
+TITLE = tc.TITLE
+
 
-llm_calc_app = gr.Blocks(css=CSS)
+llm_calc_app = gr.Blocks()
 with llm_calc_app:
 
     gr.HTML(TITLE)
 
-    ##################################################
-
     with gr.Row():
 
         #####################################
@@ -91,14 +85,14 @@ with llm_calc_app:
                     choices=langs,
                     value=[],
                     multiselect=True,
-                    label="Select Languages 🗣️"
+                    label="Languages 🗣️"
                 )
 
             with gr.Row():
                 start_date = gr.DateTime(
                     value=min_date,
                     type="string",
-                    label="Release Date Range 📅 - Start Date"
+                    label="Release Date 📅 | Start Date"
                 )
 
                 end_date = gr.DateTime(
@@ -110,17 +104,17 @@ with llm_calc_app:
             # Multiodality Select
             with gr.Row():
                 multimodal_checkbox = gr.CheckboxGroup(
-                    choices=['Image', 'Multi-Image', 'Audio', 'Video'],
+                    choices=[tc.SINGLE_IMG, tc.MULT_IMG, tc.AUDIO, tc.VIDEO],
                     value=[],
-                    label="Select Additional Modalities 📷🎧🎬",
+                    label="Additional Modalities 📷🎧🎬",
                 )
 
             # Open/Commercial Selection
             with gr.Row():
                 open_weight_checkbox = gr.CheckboxGroup(
-                    choices=['Open', 'Commercial'],
-                    value=['Open', 'Commercial'],
-                    label="Filter by Model Type 🔓 💼",
+                    choices=[tc.OPEN, tc.COMM],
+                    value=[tc.OPEN, tc.COMM],
+                    label="Model Type 🔓 💼",
                 )    
 
             # License selection
@@ -128,7 +122,7 @@ with llm_calc_app:
                 license_checkbox = gr.CheckboxGroup(
                     choices=licenses,
                     value=licenses,
-                    label="License Type 🛡️",
+                    label="License 🛡️",
                 )    
         
         #############################################################
@@ -141,7 +135,7 @@ with llm_calc_app:
                 parameter_slider = RangeSlider(
                     minimum=0, 
                     maximum=max_parameter, 
-                    label=f"Select Parameter Range 🔍 {int(min_parameters)}B - {int(max_parameter)}B+",
+                    label=f"Parameters 🔍 {int(min_parameters)}B - {int(max_parameter)}B+",
                     elem_id="double-slider-1",
                     step=parameter_step
                 )
@@ -153,7 +147,7 @@ with llm_calc_app:
                 context_slider = RangeSlider(
                     minimum=0, 
                     maximum=max_context, 
-                    label="Select Context Range (k) 📏",
+                    label="Context (k) 📏",
                     elem_id="double-slider-2",
                     step=context_step
                 )
@@ -164,7 +158,7 @@ with llm_calc_app:
                     minimum=0, 
                     maximum=max_input_price, 
                     value=(0, max_input_price), 
-                    label="Select Price range 💲/1M input tokens",
+                    label="💲/1M input tokens",
                     elem_id="double-slider-3"
                 )
             
@@ -174,7 +168,7 @@ with llm_calc_app:
                     minimum=0, 
                     maximum=max_output_price, 
                     value=(0, max_output_price), 
-                    label="Select Price range 💲/1M output tokens",
+                    label="💲/1M output tokens",
                     elem_id="double-slider-4"
                 )
        

assets/text_content.py

@@ -1,5 +1,6 @@
 import os 
 
+# Data Sources
 CLEMBENCH_RUNS_REPO = "https://raw.githubusercontent.com/clembench/clembench-runs/main/"
 REGISTRY_URL = "https://raw.githubusercontent.com/clp-research/clembench/refs/heads/refactor_model_registry/backends/model_registry.json"
 BENCHMARK_FILE = "benchmark_runs.json"
@@ -8,46 +9,31 @@ LATENCY_FOLDER = os.path.join("Addenda", "Latency")
 RESULT_FILE = "results.csv"
 LATENCY_SUFFIX = "_latency.csv"
 
-LANG_MAPPING = {
-    'el': 'Greek',
-    'id': 'Indonesian',
-    'ko': 'Korean',
-    'sv': 'Swedish',
-    'de': 'German',
-    'lv': 'Latvian',
-    'am': 'Amharic',
-    'fi': 'Finnish',
-    'da': 'Danish',
-    'pt': 'Portuguese',
-    'sw': 'Swahili',
-    'es': 'Spanish',
-    'it': 'Italian',
-    'bn': 'Bengali',
-    'nl': 'Dutch',
-    'lt': 'Lithuanian',
-    'ro': 'Romanian',
-    'sl': 'Slovenian',
-    'hu': 'Hungarian',
-    'hr': 'Croatian',
-    'vi': 'Vietnamese',
-    'hi': 'Hindi',
-    'zh': 'Chinese',
-    'pl': 'Polish',
-    'ar': 'Arabic',
-    'cs': 'Czech',
-    'sk': 'Slovak',
-    'ja': 'Japanese',
-    'no': 'Norwegian',
-    'uk': 'Ukrainian',
-    'fr': 'French',
-    'et': 'Estonian',
-    'ru': 'Russian',
-    'th': 'Thai',
-    'bg': 'Bulgarian',
-    'tr': 'Turkish',
-    'ms': 'Malay',
-    'he': 'Hebrew',
-    'tl': 'Tagalog',
-    'sr': 'Serbian',
-    'en': 'English'
-}
+# Setup Column Names
+DEFAULT_MODEL_NAME = "Unnamed: 0"
+DEFAULT_CLEMSCORE = "-, clemscore"
+
+MODEL_NAME = "Model Name"
+CLEMSCORE = "Clemscore"
+LATENCY = "Latency (s)"
+PARAMS = "Parameters (B)"
+RELEASE_DATE = 'Release Date'
+OPEN_WEIGHT = 'Open Weight'
+LANGS = "Languages"
+CONTEXT = "Context Size (k)"
+LICENSE_NAME = "License Name"
+LICENSE_URL = "License URL"
+SINGLE_IMG = "Single Image"
+MULT_IMG = "Multiple Images"
+AUDIO = "Audio"
+VIDEO = "Video"
+INPUT = "Input $/1M tokens"
+OUTPUT = "Output $/1M tokens"
+LICENSE = "License"
+TEMP_DATE = "Temp Date"
+
+# UI - HF Sapce
+OPEN = "Open-Weight"
+COMM = "Commercial"
+
+TITLE = """<h1 align="center" id="space-title"> LLM Calculator ⚖️⚡ 📏💰</h1>"""

requirements.txt

@@ -1,5 +1,4 @@
-beautifulsoup4==4.12.3
 pandas==2.2.3
 gradio_rangeslider==0.0.7
 gradio==4.44.1
-langcodes==3.5.0
+pycountry==24.6.1

src/filter_utils.py

@@ -1,19 +1,20 @@
 # Utility functions for filtering the dataframe
 
 import pandas as pd
+import assets.text_content as tc
 
 def filter_cols(df):
 
     df = df[[
-    'Model Name', 
-    'Clemscore',
-    'Input $/1M tokens', 
-    'Output $/1M tokens',
-    'Latency (s)',
-    'Context Size (k)', 
-    'Parameters (B)',
-    'Release Date', 
-    'License'
+    tc.MODEL_NAME, 
+    tc.CLEMSCORE,
+    tc.INPUT, 
+    tc.OUTPUT,
+    tc.LATENCY,
+    tc.CONTEXT, 
+    tc.PARAMS,
+    tc.RELEASE_DATE, 
+    tc.LICENSE
     ]]
     
     return df
@@ -24,73 +25,65 @@ def filter(df, language_list, parameters, input_price, output_price, multimodal,
     
 
     if not df.empty:  # Check if df is non-empty
-        df = df[df['Languages'].apply(lambda x: all(lang in x for lang in language_list))]
+        df = df[df[tc.LANGS].apply(lambda x: all(lang in x for lang in language_list))]
 
     if not df.empty:
         # Split dataframe by Open Weight
-        open_weight_true = df[df['Open Weight'] == True]
-        open_weight_false = df[df['Open Weight'] == False]
+        open_weight_true = df[df[tc.OPEN_WEIGHT] == True]
+        open_weight_false = df[df[tc.OPEN_WEIGHT] == False]
         
         # Get max parameter size for open weight models
-        max_parameter_size = open_weight_true['Parameters (B)'].max() if not open_weight_true.empty else 0
+        max_parameter_size = open_weight_true[tc.PARAMS].max() if not open_weight_true.empty else 0
         
         # Filter only the open weight models based on parameters
         if not open_weight_true.empty:
             if parameters[1] >= max_parameter_size:
                 filtered_open = open_weight_true[
-                    (open_weight_true['Parameters (B)'] >= parameters[0])
+                    (open_weight_true[tc.PARAMS] >= parameters[0])
                 ]
             else:
                 filtered_open = open_weight_true[
-                    (open_weight_true['Parameters (B)'] >= parameters[0]) & 
-                    (open_weight_true['Parameters (B)'] <= parameters[1])
+                    (open_weight_true[tc.PARAMS] >= parameters[0]) & 
+                    (open_weight_true[tc.PARAMS] <= parameters[1])
                 ]
             
             # Combine filtered open weight models with unfiltered commercial models
             df = pd.concat([filtered_open, open_weight_false])
 
     if not df.empty:  # Check if df is non-empty
-        df = df[(df['Input $/1M tokens'] >= input_price[0]) & (df['Input $/1M tokens'] <= input_price[1])]
+        df = df[(df[tc.INPUT] >= input_price[0]) & (df[tc.INPUT] <= input_price[1])]
     
     if not df.empty:  # Check if df is non-empty
-        df = df[(df['Output $/1M tokens'] >= output_price[0]) & (df['Output $/1M tokens'] <= output_price[1])]
-
-
-    print("Price")    
-    print(df)
+        df = df[(df[tc.OUTPUT] >= output_price[0]) & (df[tc.OUTPUT] <= output_price[1])]
 
     if not df.empty:  # Check if df is non-empty
-        if "Image" in multimodal:
-            df = df[df['Image'] == True]
-        if "Multi-Image" in multimodal:
-            df = df[df['Multiple Image'] == True]
-        if "Audio" in multimodal:
-            df = df[df['Audio'] == True]
-        if "Video" in multimodal:
-            df = df[df['Video'] == True]
+        if tc.SINGLE_IMG in multimodal:
+            df = df[df[tc.SINGLE_IMG] == True]
+        if tc.MULT_IMG in multimodal:
+            df = df[df[tc.MULT_IMG] == True]
+        if tc.AUDIO in multimodal:
+            df = df[df[tc.AUDIO] == True]
+        if tc.VIDEO in multimodal:
+            df = df[df[tc.VIDEO] == True]
 
     # if not df.empty:  # Check if df is non-empty
     #     df = df[(df['Context Size (k)'] >= (context[0])) & (df['Context Size (k)'] <= (context[1]))]
 
-
-    print("Modality")    
-    print(df)
-
     if not df.empty:  # Check if df is non-empty
-        if "Open" in open_weight and "Commercial" not in open_weight:
-            df = df[df['Open Weight'] == True]
-        elif "Commercial" in open_weight and "Open" not in open_weight:
-            df = df[df['Open Weight'] == False]
-        elif "Open" not in open_weight and "Commercial" not in open_weight:
+        if tc.OPEN in open_weight and tc.COMM not in open_weight:
+            df = df[df[tc.OPEN_WEIGHT] == True]
+        elif tc.COMM in open_weight and tc.OPEN not in open_weight:
+            df = df[df[tc.OPEN_WEIGHT] == False]
+        elif tc.OPEN not in open_weight and tc.COMM not in open_weight:
             # Return empty DataFrame with same columns
             df = pd.DataFrame(columns=df.columns)
         
     if not df.empty:  # Check if df is non-empty
-        df = df[df['License Name'].apply(lambda x: any(lic in x for lic in license))]
+        df = df[df[tc.LICENSE_NAME].apply(lambda x: any(lic in x for lic in license))]
 
     # Convert 'Release Date' to int temporarily
     if not df.empty:  # Check if df is non-empty
-        df['Temp Date'] = pd.to_datetime(df['Temp Date']).astype(int) // 10**9  # Convert to seconds since epoch
+        df[tc.TEMP_DATE] = pd.to_datetime(df[tc.TEMP_DATE]).astype(int) // 10**9  # Convert to seconds since epoch
 
     # Convert start and end to int (seconds since epoch)
     start = int(pd.to_datetime(start).timestamp())  
@@ -98,12 +91,10 @@ def filter(df, language_list, parameters, input_price, output_price, multimodal,
 
     # Filter based on the converted 'Release Date'
     if not df.empty:  # Check if df is non-empty
-        df = df[(df['Temp Date'] >= start) & (df['Temp Date'] <= end)]
+        df = df[(df[tc.TEMP_DATE] >= start) & (df[tc.TEMP_DATE] <= end)]
 
     df = filter_cols(df)
-    df = df.sort_values(by='Clemscore', ascending=False)
-
-    print(df)
+    df = df.sort_values(by=tc.CLEMSCORE, ascending=False)
 
     return df  # Return the filtered dataframe
     

src/process_data.py

@@ -2,9 +2,11 @@ import pandas as pd
 import json
 import os
 import pycountry
+import re
 
 from src.collect_data import fetch_version_metadata, fetch_registry_data
-from assets.text_content import LANG_MAPPING
+import assets.text_content as tc
+
 PRICING_PATH = os.path.join('assets', 'pricing.json')
 
 # Convert parameters to float, handling both B and T suffixes
@@ -64,6 +66,11 @@ def get_multimodality_field(model_data, field):
     except:
         return False
 
+def clean_model_name(model_name: str) -> str:
+    """Clean model name by removing temperature suffix pattern."""
+    # Match pattern like -t0.0--, -t0.7--, -t1.0--, etc.
+    pattern = r'-t[0-1]\.[0-9]--'
+    return re.split(pattern, model_name)[0]
 
 def merge_data():
 
@@ -73,10 +80,10 @@ def merge_data():
         pricing_data = json.load(f)
 
     # Ensure the unnamed column is renamed to 'model'
-    mm_result_df.rename(columns={'Unnamed: 0': 'model', '-, clemscore': 'clemscore'}, inplace=True)
-    text_result_df.rename(columns={'Unnamed: 0': 'model', '-, clemscore': 'clemscore'}, inplace=True)
-    mm_result_df['model'] = mm_result_df['model'].str.split('-t0.0--').str[0]
-    text_result_df['model'] = text_result_df['model'].str.split('-t0.0--').str[0] # Bug in get_latency.py, split by -t0.0 instead of -t (gpt-3.5-turbo/gpt-4-turbo breaks) 
+    mm_result_df.rename(columns={tc.DEFAULT_MODEL_NAME: 'model', tc.DEFAULT_CLEMSCORE: 'clemscore'}, inplace=True)
+    text_result_df.rename(columns={tc.DEFAULT_MODEL_NAME: 'model', tc.DEFAULT_CLEMSCORE: 'clemscore'}, inplace=True)
+    mm_result_df['model'] = mm_result_df['model'].apply(clean_model_name)
+    text_result_df['model'] = text_result_df['model'].apply(clean_model_name)
 
     # Merge datasets to compute average values
     avg_latency_df = pd.concat([mm_latency_df, text_latency_df], axis=0).groupby('model')['latency'].mean().reset_index()
@@ -116,20 +123,20 @@ def merge_data():
     
     # Update column renaming
     merged_df = merged_df.rename(columns={
-        'model': 'Model Name',
-        'latency': 'Latency (s)',
-        'clemscore': 'Clemscore',
-        'parameters': 'Parameters (B)',
-        'release_date': 'Release Date',
-        'open_weight': 'Open Weight',
-        'languages': 'Languages',
-        'context_size': 'Context Size (k)',
-        'license_name': 'License Name',
-        'license_url': 'License URL',
-        'single_image': 'Single Image',
-        'multiple_images': 'Multiple Images',
-        'audio': 'Audio',
-        'video': 'Video'
+        'model': tc.MODEL_NAME,
+        'latency': tc.LATENCY,
+        'clemscore': tc.CLEMSCORE,
+        'parameters': tc.PARAMS,
+        'release_date': tc.RELEASE_DATE,
+        'open_weight': tc.OPEN_WEIGHT,
+        'languages': tc.LANGS,
+        'context_size': tc.CONTEXT,
+        'license_name': tc.LICENSE_NAME,
+        'license_url': tc.LICENSE_URL,
+        'single_image': tc.SINGLE_IMG,
+        'multiple_images': tc.MULT_IMG,
+        'audio': tc.AUDIO,
+        'video': tc.VIDEO
     })
     
     # Convert pricing_data list to DataFrame
@@ -149,27 +156,27 @@ def merge_data():
     # Drop duplicate model column and rename price columns
     merged_df = merged_df.drop('model_id', axis=1)
     merged_df = merged_df.rename(columns={
-        'input': 'Input $/1M tokens',
-        'output': 'Output $/1M tokens'
+        'input': tc.INPUT,
+        'output': tc.OUTPUT
     })
     
     # Fill NaN values with 0.0 for pricing columns
-    merged_df['Input $/1M tokens'] = merged_df['Input $/1M tokens'].fillna(0.0)
-    merged_df['Output $/1M tokens'] = merged_df['Output $/1M tokens'].fillna(0.0)
+    merged_df[tc.INPUT] = merged_df[tc.INPUT].fillna(0.0)
+    merged_df[tc.OUTPUT] = merged_df[tc.OUTPUT].fillna(0.0)
     
     # Convert parameters and set to None for commercial models
-    merged_df['Parameters (B)'] = merged_df.apply(
-        lambda row: None if not row['Open Weight'] else convert_parameters(row['Parameters (B)']), 
+    merged_df[tc.PARAMS] = merged_df.apply(
+        lambda row: None if not row[tc.OPEN_WEIGHT] else convert_parameters(row[tc.PARAMS]), 
         axis=1
     )
 
-    merged_df['License'] = merged_df.apply(lambda row: f'<a href="{row["License URL"]}" style="color: blue;">{row["License Name"]}</a>', axis=1)
-    merged_df['Temp Date'] = merged_df['Release Date']
+    merged_df[tc.LICENSE] = merged_df.apply(lambda row: f'<a href="{row[tc.LICENSE_URL]}" style="color: blue;">{row[tc.LICENSE_NAME]}</a>', axis=1)
+    merged_df[tc.TEMP_DATE] = merged_df[tc.RELEASE_DATE]
 
-    merged_df['Languages'] = merged_df['Languages'].apply(map_languages)
+    merged_df[tc.LANGS] = merged_df[tc.LANGS].apply(map_languages)
 
     # Sort by Clemscore in descending order
-    merged_df = merged_df.sort_values(by='Clemscore', ascending=False)
+    merged_df = merged_df.sort_values(by=tc.CLEMSCORE, ascending=False)
     
     # Drop model_name column
     merged_df.drop(columns=['model_name'], inplace=True)