website

spit-fight-website/.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

spit-fight-website/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

spit-fight-website/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7" project-jdk-type="Python SDK" />
+</project>

spit-fight-website/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/spit-fight-website.iml" filepath="$PROJECT_DIR$/.idea/spit-fight-website.iml" />
+    </modules>
+  </component>
+</project>

spit-fight-website/.idea/spit-fight-website.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

spit-fight-website/leaderboard.py

@@ -0,0 +1,94 @@
+import gradio as gr
+import requests
+import pandas as pd
+from huggingface_hub.hf_api import SpaceInfo
+import matplotlib.pyplot as plt
+import plotly.express as px
+
+model_perf_table = 'tables/test.csv'
+logo_path = 'img/image.png'
+
+
+def get_blocks_party_spaces():
+    df = pd.read_csv(model_perf_table)
+    df = df.sort_values(by=['score'],ascending=False)
+    return df
+
+
+def get_blocks_party_spaces_with_formula(formula=None):
+    # get the dataframe
+    df = get_blocks_party_spaces()
+    if formula:
+        try:
+            df[str(formula)] = df.eval(formula)
+        except:
+            pass # Handle this error properly in your code
+    return df
+
+def create_scatter(x, y, z):
+    df = get_blocks_party_spaces()
+
+    if z is None or z == 'None':
+        fig = plt.figure()
+        ax = fig.add_subplot()
+        scatter = ax.scatter(list(df[x]),list(df[y]))
+        for i, label in enumerate(list(df['model'])):
+            ax.text(list(df[x])[i],list(df[y])[i],str(label))
+        ax.set_xlabel(x)
+        ax.set_ylabel(y)
+    else:
+        fig = px.scatter_3d(df, x=x, y=y, z=z, text=df['model'])
+
+        # Set axis labels and title
+        fig.update_layout(scene=dict(
+            xaxis_title=x,
+            yaxis_title=y,
+            zaxis_title=z,
+        ),
+            title='3D Scatter Plot'
+        )
+
+    return fig
+
+block = gr.Blocks()
+with block:
+    # gr.outputs.HTML(f'<img src="{logo_path}" alt="logo" height="1000px">')
+    # img = gr.Image(logo_path,shape=[1,2]).style( rounded=False)
+
+    gr.Markdown(f"""
+                # 🦙💦SpitFight - Leaderboard for LLM 
+                """)
+    with gr.Tabs():
+        with gr.TabItem("Leaderboard"):
+            with gr.Row():
+                data = gr.outputs.Dataframe(type="pandas")
+            with gr.Row():
+                formula_input = gr.inputs.Textbox(lines=1, label="User Designed Column", placeholder = 'e.g. verbosity/latency')
+                data_run = gr.Button("Add To Table")
+                data_run.click(get_blocks_party_spaces_with_formula, inputs=formula_input, outputs=data)
+            # running the function on page load in addition to when the button is clicked
+            with gr.Row():
+                with gr.Column():
+                    scatter_input = [gr.inputs.Dropdown(choices=get_blocks_party_spaces().columns.tolist()[1:], label="X-axis"),
+                         gr.inputs.Dropdown(choices=get_blocks_party_spaces().columns.tolist()[1:], label="Y-axis"),
+                         gr.inputs.Dropdown(choices=[None]+get_blocks_party_spaces().columns.tolist()[1:], label="Z-axis (Optional)")]
+                    fig_run = gr.Button("Generate Figure")
+
+                with gr.Column():
+                    gen_figure = gr.Plot()# gr.outputs.Image(type="pil")
+                    fig_run.click(create_scatter, inputs=scatter_input, outputs=gen_figure)
+
+
+        with gr.TabItem("About"):
+            gr.Markdown(f"""
+                        ## Metrics: 
+                        - **Human Score**: The average score given by human evaluators.
+                        - **Throughput**: The number of tokens generated per second.
+                        - **Verbosity**: The average number of generated tokens in the model's response.
+                        - **Latency**: The average time it takes for the model to generate a response.
+                        - **Memory**: The base memory usage of the model.
+                        """)
+
+    block.load(get_blocks_party_spaces_with_formula, inputs=None, outputs=data)
+
+block.launch()

spit-fight-website/requirements.txt

@@ -0,0 +1,2 @@
+gradio
+plotly

spit-fight-website/tables/test.csv

@@ -0,0 +1,6 @@
+model,score,throughput,verbosity,latency,energy,memory
+vicuna-13B,1054,19.47,178.71,10.37,12,26232
+vicuna-7B,1007,31.26,217.33,7.71,32,14087
+llama-13B,854,18.49,390.9,22.74,24,26233
+t5-3B,941,31.98,131.12,6.09,4,8031
+rwkv-7B,928,23.09,239.98,9.62,12,15790