update

app.py

@@ -195,7 +195,6 @@ with llm_calc_app:
                                 elem_id="text-leaderboard-table",
                                 interactive=False,
                                 visible=True,
-                                height=800,
                                 datatype=['html', 'number', 'number', 'date', 'number', 'number', 'number', 'number', 'html']
                             )
 

assets/pricing.json

@@ -0,0 +1,212 @@
+[
+    {
+        "model_id": "gpt-4-1106-vision-preview",
+        "input": "10$",
+        "output": "30$"
+    },
+    {
+        "model_id": "gpt-4o-2024-05-13",
+        "input": "5$",
+        "output": "15$"
+    },
+    {
+        "model_id": "gpt-4o-2024-08-06",
+        "input": "3.750$",
+        "output": "15$"
+    },
+    {
+        "model_id": "gpt-4o-mini-2024-07-18",
+        "input": "0.300$",
+        "output": "1.200$"
+    },
+    {
+        "model_id": "gpt-4-turbo-2024-04-09",
+        "input": "10$",
+        "output": "30$"
+    },
+    {
+        "model_id": "gpt-4-1106-preview",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "gpt-4-0125-preview",
+        "input": "10$",
+        "output": "30$"
+    },
+    {
+        "model_id": "o1-preview-2024-09-12",
+        "input": "15$",
+        "output": "60$"
+    },
+    {
+        "model_id": "o1-mini-2024-09-12",
+        "input": "3$",
+        "output": "12$"
+    },
+    {
+        "model_id": "gpt-3.5-turbo-0125",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gpt-4-0613",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "gpt-4-0314",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "gpt-3.5-turbo-1106",
+        "input": "1$",
+        "output": "2$"
+    },
+    {
+        "model_id": "gpt-3.5-turbo-0613",
+        "input": "1.5$",
+        "output": "2$"
+    },
+    {
+        "model_id": "command",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "command-light",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-v1.3",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-v1.3-100k",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-instant-1.2",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "claude-2",
+        "input": "8$",
+        "output": "24$"
+    },
+    {
+        "model_id": "claude-2.1",
+        "input": "8$",
+        "output": "24$"
+    },
+    {
+        "model_id": "claude-3-opus-20240229",
+        "input": "15$",
+        "output": "75$"
+    },
+    {
+        "model_id": "claude-3-sonnet-20240229",
+        "input": "3$",
+        "output": "15$"
+    },
+    {
+        "model_id": "claude-3-haiku-20240307",
+        "input": "0.25$",
+        "output": "1.25$"
+    },
+    {
+        "model_id": "claude-3-5-sonnet-20240620",
+        "input": "3$",
+        "output": "15$"
+    },
+    {
+        "model_id": "claude-3-5-haiku-20241022",
+        "input": "0.8$",
+        "output": "4$"
+    },
+    {
+        "model_id": "claude-3-5-sonnet-20241022",
+        "input": "3$",
+        "output": "15$"
+    },
+    {
+        "model_id": "gemini-1.0-pro-001",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gemini-1.0-pro-002",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gemini-1.0-pro-vision-latest",
+        "input": "0.5$",
+        "output": "1.5$"
+    },
+    {
+        "model_id": "gemini-1.5-flash-001",
+        "input": "0.075$",
+        "output": "0.3$"
+    },
+    {
+        "model_id": "gemini-1.5-pro-001",
+        "input": "1.25$",
+        "output": "5$"
+    },
+    {
+        "model_id": "gemini-1.5-pro-002",
+        "input": "1.25$",
+        "output": "5$"
+    },
+    {
+        "model_id": "gemini-1.5-flash-002",
+        "input": "0.075$",
+        "output": "0.3$"
+    },
+    {
+        "model_id": "gemini-1.5-flash-8b-001",
+        "input": "0.0375$",
+        "output": "0.15$"
+    },
+    {
+        "model_id": "gemini-2.0-flash-exp",
+        "input": "0$",
+        "output": "0$"
+    },
+    {
+        "model_id": "luminous-supreme-control",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-supreme",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-extended",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-base",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-base",
+        "input": "",
+        "output": ""
+    },
+    {
+        "model_id": "luminous-base",
+        "input": "",
+        "output": ""
+    }
+]

assets/text_content.py

@@ -0,0 +1,3 @@
+CLEMBENCH_RUNS_REPO = "https://raw.githubusercontent.com/clembench/clembench-runs/main/"
+REGISTRY_URL = "https://raw.githubusercontent.com/clp-research/clembench/refs/heads/refactor_model_registry/backends/model_registry.json"
+BENCHMARK_FILE = "benchmark_runs.json"

prepare_path.sh

@@ -0,0 +1,2 @@
+#!/bash/bin
+export PYTHONPATH=.:$PYTHONPATH

requirements.txt

@@ -1,4 +1,5 @@
 beautifulsoup4==4.12.3
 pandas==2.2.3
 gradio_rangeslider==0.0.7
-gradio==4.44.1
+gradio==5.9.0
+langcodes==3.5.0

src/collect_data.py

@@ -0,0 +1,18 @@
+"""
+Collect data from the multiple sources and create a base datafranme for the LLMCalculator table
+Latency - https://github.com/clembench/clembench-runs/tree/main/Addenda/Latency
+Pricing - pricing.json
+Model info - https://github.com/kushal-10/clembench/blob/feat/registry/backends/model_registry_updated.json
+"""
+
+import pandas as pd
+import json
+import requests
+from assets.text_content import REPO, REGISTRY_URL, BENCHMARK_FILE
+
+response = requests.get(REGISTRY_URL)
+
+# Check if the JSON file request was successful
+if response.status_code != 200:
+    print(f"Failed to read JSON file - {REGISTRY_URL}. Status Code: {response.status_code}")
+