2024-02-22 14:10:21 Publish script update

app.py

@@ -57,10 +57,10 @@ output_types_df = pd.DataFrame(
     {"Output Type": queries_config.query_template.keys(), "Added text": queries_config.query_template.values()}
 )
 
-summary_df: pd.DataFrame = pd.read_csv("data/2024-02-05 23:33:22.947120_summary.csv")
+summary_df: pd.DataFrame = pd.read_csv("data/summary.csv")
 time_of_day_comparison_df = pd.read_csv("data/time_of_day_comparison.csv")
 general_plots = pd.read_csv("data/general_plots.csv")
-model_costs_df = pd.read_csv("data/2024-02-05 12:03:45.281624_model_costs.csv")
+model_costs_df = pd.read_csv("data/model_costs.csv")
 time_of_day_plots = pd.read_csv("data/time_of_day_plots.csv")
 output_plots = pd.read_csv("data/output_plots.csv")
 
@@ -97,6 +97,13 @@ def collapse_output_method_toggle():
         button_text = "Un-collapse output method"
     return dataframes()[0], button_text
 
+def filter_dataframe(df, searched_model_names):
+    if not searched_model_names:
+        return df
+    filter_series = df.model == "" # False values
+    for n in searched_model_names:
+        filter_series = filter_series | df.model.str.lower().str.contains(n)
+    return df[filter_series]
 
 def dataframes():
     global collapse_languages, collapse_output_method, searched_query, summary_df, time_of_day_comparison_df, model_costs_df
@@ -117,9 +124,7 @@ def dataframes():
     searched_model_names = [n for n in searched_model_names if n]
 
     def for_dataframe(df):
-        if not searched_model_names:
-            return dataframe_style(df)
-        return dataframe_style(pd.concat(df[df.model.str.lower().str.contains(n)] for n in searched_model_names))
+        return dataframe_style(filter_dataframe(df, searched_model_names))
 
     return (
         for_dataframe(summary_df_processed),
@@ -172,18 +177,13 @@ def filter_plots(searched_query: str):
     searched_model_names = [n.lower().strip() for n in searched_model_names]
     searched_model_names = [n for n in searched_model_names if n]
 
-    def filter_dataframe(df):
-        if not searched_model_names:
-            return df
-        return pd.concat(df[df.model.str.lower().str.contains(n)] for n in searched_model_names)
-
     results = []
     for plot_display, plot, row in plots:
         visible = True
         if "df" in row and pd.notna(row["df"]):
             buffer = io.StringIO(row["df"])
             df = pd.read_csv(buffer)
-            df = filter_dataframe(df)
+            df = filter_dataframe(df, searched_model_names)
             plot = px.bar(df, **json.loads(row["arguments"]))
             plot.update_layout(autosize=True)
         elif "for model" in row["header"] and searched_model_names:
@@ -316,4 +316,4 @@ Mainly OpenAI's GPT models and Google's Palm 2.
         api_name="collapse_output_method_toggle",
     )
 
-demo.launch(share=True)
+demo.launch()

data/general_plots.csv

@@ -10,7 +10,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.1446]),
+              'x': array([17.4942]),
               'xaxis': 'x',
               'y': array(['gpt-4'], dtype=object),
               'yaxis': 'y'},
@@ -24,7 +24,7 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.056]),
+              'x': array([8.7136]),
               'xaxis': 'x',
               'y': array(['gpt-4-turbo'], dtype=object),
               'yaxis': 'y'},
@@ -38,119 +38,338 @@ execution_costs,./html/plots/execution_costs.html,"Figure({
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.00442]),
+              'x': array([0.74798]),
               'xaxis': 'x',
               'y': array(['gpt-3.5-turbo'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'llama-2-70b-chat',
+              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'Mixtral-8x7B-Instruct-v0.1',
+              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.74277]),
+              'xaxis': 'x',
+              'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.67250116]),
+              'xaxis': 'x',
+              'y': array(['zephyr-7b-beta'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'llama-2-70b-chat',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
               'name': 'llama-2-70b-chat',
               'offsetgroup': 'llama-2-70b-chat',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.002808]),
+              'x': array([0.665964]),
               'xaxis': 'x',
               'y': array(['llama-2-70b-chat'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
-              'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
-              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
-              'name': 'Mixtral-8x7B-Instruct-v0.1',
-              'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+              'legendgroup': '01-ai Yi Chat (34B)',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': '01-ai Yi Chat (34B)',
+              'offsetgroup': '01-ai Yi Chat (34B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.00207]),
+              'x': array([0.405168]),
               'xaxis': 'x',
-              'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+              'y': array(['01-ai Yi Chat (34B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.23022898]),
+              'xaxis': 'x',
+              'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
               'legendgroup': 'gemini-pro',
-              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
               'name': 'gemini-pro',
               'offsetgroup': 'gemini-pro',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.001195]),
+              'x': array([0.178845]),
               'xaxis': 'x',
               'y': array(['gemini-pro'], dtype=object),
               'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'Snorkel Mistral PairRM DPO (7B)',
+              'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.15948]),
+              'xaxis': 'x',
+              'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
+              'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
               'legendgroup': 'chat-bison (PaLM 2)',
-              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
               'name': 'chat-bison (PaLM 2)',
               'offsetgroup': 'chat-bison (PaLM 2)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.001075]),
+              'x': array([0.158515]),
               'xaxis': 'x',
               'y': array(['chat-bison (PaLM 2)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
               'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
-              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
               'name': 'chat-bison-32k (PaLM 2 32K)',
               'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.001025]),
+              'x': array([0.151035]),
               'xaxis': 'x',
               'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
               'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.148248]),
+              'xaxis': 'x',
+              'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Chronos Hermes (13B)',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+              'name': 'Chronos Hermes (13B)',
+              'offsetgroup': 'Chronos Hermes (13B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.141156]),
+              'xaxis': 'x',
+              'y': array(['Chronos Hermes (13B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'WizardLM v1.2 (13B)',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'WizardLM v1.2 (13B)',
+              'offsetgroup': 'WizardLM v1.2 (13B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.131244]),
+              'xaxis': 'x',
+              'y': array(['WizardLM v1.2 (13B)'], dtype=object),
+              'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
               'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
-              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
               'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
               'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([0.00062769]),
+              'x': array([0.11099814]),
               'xaxis': 'x',
               'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'LLaMA-2 Chat (7B)',
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+              'name': 'LLaMA-2 Chat (7B)',
+              'offsetgroup': 'LLaMA-2 Chat (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.103212]),
+              'xaxis': 'x',
+              'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
+              'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+              'name': 'Upstage SOLAR Instruct v1 (11B)',
+              'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.10254]),
+              'xaxis': 'x',
+              'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Vicuna v1.5 (7B)',
+              'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+              'name': 'Vicuna v1.5 (7B)',
+              'offsetgroup': 'Vicuna v1.5 (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.097484]),
+              'xaxis': 'x',
+              'y': array(['Vicuna v1.5 (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Qwen 1.5 Chat (7B)',
+              'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+              'name': 'Qwen 1.5 Chat (7B)',
+              'offsetgroup': 'Qwen 1.5 Chat (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.092256]),
+              'xaxis': 'x',
+              'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+              'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'OpenHermes-2.5-Mistral (7B)',
+              'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.089096]),
+              'xaxis': 'x',
+              'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'Falcon Instruct (7B)',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'Falcon Instruct (7B)',
+              'offsetgroup': 'Falcon Instruct (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.07428]),
+              'xaxis': 'x',
+              'y': array(['Falcon Instruct (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Execution cost ($)=%{x}<extra></extra>',
+              'legendgroup': 'RedPajama-INCITE Chat (7B)',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'RedPajama-INCITE Chat (7B)',
+              'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([0.07172]),
+              'xaxis': 'x',
+              'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
               'yaxis': 'y'}],
     'layout': {'barmode': 'relative',
                'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
                'template': '...',
-               'title': {'text': 'Costs of execution of 20 test queries per model'},
+               'title': {'text': 'Costs of execution of 5560 test queries per model'},
                'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
                'yaxis': {'anchor': 'x',
-                         'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
-                                           chat-bison-32k (PaLM 2 32K), chat-bison
-                                           (PaLM 2), gemini-pro,
+                         'categoryarray': [RedPajama-INCITE Chat (7B), Falcon
+                                           Instruct (7B), OpenHermes-2.5-Mistral
+                                           (7B), Qwen 1.5 Chat (7B), Vicuna v1.5
+                                           (7B), Upstage SOLAR Instruct v1 (11B),
+                                           LLaMA-2 Chat (7B),
+                                           TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+                                           WizardLM v1.2 (13B), Chronos Hermes
+                                           (13B), Mistral (7B) Instruct v0.2
+                                           (Together AI), chat-bison-32k (PaLM 2
+                                           32K), chat-bison (PaLM 2), Snorkel
+                                           Mistral PairRM DPO (7B), gemini-pro,
+                                           Mistral-7B-Instruct-v0.2, 01-ai Yi Chat
+                                           (34B), llama-2-70b-chat, zephyr-7b-beta,
                                            Mixtral-8x7B-Instruct-v0.1,
-                                           llama-2-70b-chat, gpt-3.5-turbo,
-                                           gpt-4-turbo, gpt-4],
+                                           gpt-3.5-turbo, gpt-4-turbo, gpt-4],
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
-})",Costs of execution of 20 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.1446],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.056],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00442],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.002808],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0020700000000000002],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0011949999999999999],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.001075],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0010249999999999999],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0006276866594950359],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
-2,0.1446,gpt-4
-1,0.056,gpt-4-turbo
-0,0.00442,gpt-3.5-turbo
-3,0.002808,llama-2-70b-chat
-4,0.0020700000000000002,Mixtral-8x7B-Instruct-v0.1
-8,0.0011949999999999999,gemini-pro
-9,0.001075,chat-bison (PaLM 2)
-10,0.0010249999999999999,chat-bison-32k (PaLM 2 32K)
-7,0.0006276866594950359,TinyLlama/TinyLlama-1.1B-Chat-v1.0
+})",Costs of execution of 5560 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[17.4942],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8.7136],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.74798],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.7427699999999999],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.6725011554823982],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.665964],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.40516800000000003],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.23022897595829436],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.178845],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.15948],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.15851500000000002],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.151035],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.14824800000000002],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.141156],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.131244],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.11099814383188883],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.103212],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.10253999999999999],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.097484],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.092256],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.08909600000000001],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.07428],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.07172],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""OpenHermes-2.5-Mistral (7B)"",""Qwen 1.5 Chat (7B)"",""Vicuna v1.5 (7B)"",""Upstage SOLAR Instruct v1 (11B)"",""LLaMA-2 Chat (7B)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""WizardLM v1.2 (13B)"",""Chronos Hermes (13B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""Snorkel Mistral PairRM DPO (7B)"",""gemini-pro"",""Mistral-7B-Instruct-v0.2"",""01-ai Yi Chat (34B)"",""llama-2-70b-chat"",""zephyr-7b-beta"",""Mixtral-8x7B-Instruct-v0.1"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 5560 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 5560 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
+2,17.4942,gpt-4
+1,8.7136,gpt-4-turbo
+0,0.74798,gpt-3.5-turbo
+4,0.7427699999999999,Mixtral-8x7B-Instruct-v0.1
+5,0.6725011554823982,zephyr-7b-beta
+3,0.665964,llama-2-70b-chat
+11,0.40516800000000003,01-ai Yi Chat (34B)
+6,0.23022897595829436,Mistral-7B-Instruct-v0.2
+8,0.178845,gemini-pro
+43,0.15948,Snorkel Mistral PairRM DPO (7B)
+9,0.15851500000000002,chat-bison (PaLM 2)
+10,0.151035,chat-bison-32k (PaLM 2 32K)
+26,0.14824800000000002,Mistral (7B) Instruct v0.2 (Together AI)
+12,0.141156,Chronos Hermes (13B)
+55,0.131244,WizardLM v1.2 (13B)
+7,0.11099814383188883,TinyLlama/TinyLlama-1.1B-Chat-v1.0
+24,0.103212,LLaMA-2 Chat (7B)
+56,0.10253999999999999,Upstage SOLAR Instruct v1 (11B)
+17,0.097484,Vicuna v1.5 (7B)
+40,0.092256,Qwen 1.5 Chat (7B)
+46,0.08909600000000001,OpenHermes-2.5-Mistral (7B)
+48,0.07428,Falcon Instruct (7B)
+51,0.07172,RedPajama-INCITE Chat (7B)
 "
 model_sizes,./html/plots/model_sizes.html,"Figure({
     'data': [{'alignmentgroup': 'True',
@@ -183,59 +402,59 @@ model_sizes,./html/plots/model_sizes.html,"Figure({
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
-              'legendgroup': 'WizardLM v1.2 (13B)',
+              'legendgroup': '01-ai Yi Chat (34B)',
               'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
-              'name': 'WizardLM v1.2 (13B)',
-              'offsetgroup': 'WizardLM v1.2 (13B)',
+              'name': '01-ai Yi Chat (34B)',
+              'offsetgroup': '01-ai Yi Chat (34B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([13.]),
+              'x': array([34.]),
               'xaxis': 'x',
-              'y': array(['WizardLM v1.2 (13B)'], dtype=object),
+              'y': array(['01-ai Yi Chat (34B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
-              'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
+              'legendgroup': 'WizardLM v1.2 (13B)',
               'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
-              'name': 'Upstage SOLAR Instruct v1 (11B)',
-              'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
+              'name': 'WizardLM v1.2 (13B)',
+              'offsetgroup': 'WizardLM v1.2 (13B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([11.]),
+              'x': array([13.]),
               'xaxis': 'x',
-              'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
+              'y': array(['WizardLM v1.2 (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
-              'legendgroup': 'zephyr-7b-beta',
+              'legendgroup': 'Chronos Hermes (13B)',
               'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
-              'name': 'zephyr-7b-beta',
-              'offsetgroup': 'zephyr-7b-beta',
+              'name': 'Chronos Hermes (13B)',
+              'offsetgroup': 'Chronos Hermes (13B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([7.]),
+              'x': array([13.]),
               'xaxis': 'x',
-              'y': array(['zephyr-7b-beta'], dtype=object),
+              'y': array(['Chronos Hermes (13B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
-              'legendgroup': 'Mistral-7B-Instruct-v0.2',
+              'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
               'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
-              'name': 'Mistral-7B-Instruct-v0.2',
-              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+              'name': 'Upstage SOLAR Instruct v1 (11B)',
+              'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
-              'x': array([7.]),
+              'x': array([11.]),
               'xaxis': 'x',
-              'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+              'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
@@ -253,17 +472,17 @@ model_sizes,./html/plots/model_sizes.html,"Figure({
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
-              'legendgroup': 'LLaMA-2 Chat (7B)',
+              'legendgroup': 'Mistral-7B-Instruct-v0.2',
               'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
-              'name': 'LLaMA-2 Chat (7B)',
-              'offsetgroup': 'LLaMA-2 Chat (7B)',
+              'name': 'Mistral-7B-Instruct-v0.2',
+              'offsetgroup': 'Mistral-7B-Instruct-v0.2',
               'orientation': 'h',
               'showlegend': True,
               'textposition': 'auto',
               'type': 'bar',
               'x': array([7.]),
               'xaxis': 'x',
-              'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
+              'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
@@ -295,8 +514,50 @@ model_sizes,./html/plots/model_sizes.html,"Figure({
               'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
-              'legendgroup': 'RedPajama-INCITE Chat (7B)',
+              'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
               'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+              'name': 'Snorkel Mistral PairRM DPO (7B)',
+              'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([7.]),
+              'xaxis': 'x',
+              'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
+              'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'name': 'OpenHermes-2.5-Mistral (7B)',
+              'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([7.]),
+              'xaxis': 'x',
+              'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
+              'legendgroup': 'Falcon Instruct (7B)',
+              'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+              'name': 'Falcon Instruct (7B)',
+              'offsetgroup': 'Falcon Instruct (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([7.]),
+              'xaxis': 'x',
+              'y': array(['Falcon Instruct (7B)'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
+              'legendgroup': 'RedPajama-INCITE Chat (7B)',
+              'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
               'name': 'RedPajama-INCITE Chat (7B)',
               'offsetgroup': 'RedPajama-INCITE Chat (7B)',
               'orientation': 'h',
@@ -307,10 +568,38 @@ model_sizes,./html/plots/model_sizes.html,"Figure({
               'xaxis': 'x',
               'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
               'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
+              'legendgroup': 'zephyr-7b-beta',
+              'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+              'name': 'zephyr-7b-beta',
+              'offsetgroup': 'zephyr-7b-beta',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([7.]),
+              'xaxis': 'x',
+              'y': array(['zephyr-7b-beta'], dtype=object),
+              'yaxis': 'y'},
+             {'alignmentgroup': 'True',
+              'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
+              'legendgroup': 'LLaMA-2 Chat (7B)',
+              'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+              'name': 'LLaMA-2 Chat (7B)',
+              'offsetgroup': 'LLaMA-2 Chat (7B)',
+              'orientation': 'h',
+              'showlegend': True,
+              'textposition': 'auto',
+              'type': 'bar',
+              'x': array([7.]),
+              'xaxis': 'x',
+              'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
+              'yaxis': 'y'},
              {'alignmentgroup': 'True',
               'hovertemplate': 'Model=%{y}<br>Model size (B params)=%{x}<extra></extra>',
               'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
-              'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+              'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
               'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
               'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
               'orientation': 'h',
@@ -328,29 +617,38 @@ model_sizes,./html/plots/model_sizes.html,"Figure({
                'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}},
                'yaxis': {'anchor': 'x',
                          'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
-                                           RedPajama-INCITE Chat (7B), Qwen 1.5
-                                           Chat (7B), Mistral (7B) Instruct v0.2
-                                           (Together AI), LLaMA-2 Chat (7B), Vicuna
-                                           v1.5 (7B), Mistral-7B-Instruct-v0.2,
-                                           zephyr-7b-beta, Upstage SOLAR Instruct
-                                           v1 (11B), WizardLM v1.2 (13B),
+                                           LLaMA-2 Chat (7B), zephyr-7b-beta,
+                                           RedPajama-INCITE Chat (7B), Falcon
+                                           Instruct (7B), OpenHermes-2.5-Mistral
+                                           (7B), Snorkel Mistral PairRM DPO (7B),
+                                           Qwen 1.5 Chat (7B), Mistral (7B)
+                                           Instruct v0.2 (Together AI),
+                                           Mistral-7B-Instruct-v0.2, Vicuna v1.5
+                                           (7B), Upstage SOLAR Instruct v1 (11B),
+                                           Chronos Hermes (13B), WizardLM v1.2
+                                           (13B), 01-ai Yi Chat (34B),
                                            Mixtral-8x7B-Instruct-v0.1,
                                            llama-2-70b-chat],
                          'categoryorder': 'array',
                          'domain': [0.0, 1.0],
                          'title': {'text': 'Model'}}}
-})",Model sizes in billions of parameters,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[70.0],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[56.0],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[13.0],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[11.0],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.1],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""RedPajama-INCITE Chat (7B)"",""Qwen 1.5 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""LLaMA-2 Chat (7B)"",""Vicuna v1.5 (7B)"",""Mistral-7B-Instruct-v0.2"",""zephyr-7b-beta"",""Upstage SOLAR Instruct v1 (11B)"",""WizardLM v1.2 (13B)"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}","{""x"": ""model_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model sizes in billions of parameters"", ""labels"": {""model"": ""Model"", ""model_sizes"": ""Model size (B params)""}}",",model_sizes,model
+})",Model sizes in billions of parameters,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[70.0],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[56.0],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[34.0],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[13.0],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[13.0],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[11.0],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.1],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""LLaMA-2 Chat (7B)"",""zephyr-7b-beta"",""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""OpenHermes-2.5-Mistral (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""Qwen 1.5 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""Mistral-7B-Instruct-v0.2"",""Vicuna v1.5 (7B)"",""Upstage SOLAR Instruct v1 (11B)"",""Chronos Hermes (13B)"",""WizardLM v1.2 (13B)"",""01-ai Yi Chat (34B)"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}","{""x"": ""model_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model sizes in billions of parameters"", ""labels"": {""model"": ""Model"", ""model_sizes"": ""Model size (B params)""}}",",model_sizes,model
 0,70.0,llama-2-70b-chat
 1,56.0,Mixtral-8x7B-Instruct-v0.1
-10,13.0,WizardLM v1.2 (13B)
-11,11.0,Upstage SOLAR Instruct v1 (11B)
-2,7.0,zephyr-7b-beta
+5,34.0,01-ai Yi Chat (34B)
+15,13.0,WizardLM v1.2 (13B)
+6,13.0,Chronos Hermes (13B)
+16,11.0,Upstage SOLAR Instruct v1 (11B)
+7,7.0,Vicuna v1.5 (7B)
 3,7.0,Mistral-7B-Instruct-v0.2
-5,7.0,Vicuna v1.5 (7B)
-6,7.0,LLaMA-2 Chat (7B)
-7,7.0,Mistral (7B) Instruct v0.2 (Together AI)
-8,7.0,Qwen 1.5 Chat (7B)
-9,7.0,RedPajama-INCITE Chat (7B)
+9,7.0,Mistral (7B) Instruct v0.2 (Together AI)
+10,7.0,Qwen 1.5 Chat (7B)
+11,7.0,Snorkel Mistral PairRM DPO (7B)
+12,7.0,OpenHermes-2.5-Mistral (7B)
+13,7.0,Falcon Instruct (7B)
+14,7.0,RedPajama-INCITE Chat (7B)
+2,7.0,zephyr-7b-beta
+8,7.0,LLaMA-2 Chat (7B)
 4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
 "
 model_context_lengths,./html/plots/model_context_lengths.html,"Figure({

data/model_costs.csv

@@ -0,0 +1,58 @@
+model,provider,cost_per_token,provider_pricing
+gpt-3.5-turbo,OpenAI,"$1 / 1M input tokens, $2 / 1M output tokens","$1 / 1M input tokens, $2 / 1M output tokens"
+gpt-4-turbo,OpenAI,"$10 / 1M input tokens, $30 / 1M output tokens","$10 / 1M input tokens, $30 / 1M output tokens"
+gpt-4,OpenAI,"$30 / 1M input tokens, $60 / 1M output tokens","$30 / 1M input tokens, $60 / 1M output tokens"
+llama-2-70b-chat,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Mixtral-8x7B-Instruct-v0.1,Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+zephyr-7b-beta,Hugging Face Inference Endpoint,$0.15 / 1M tokens,$1.3 / hour
+Mistral-7B-Instruct-v0.2,Hugging Face Inference Endpoint,$0.14 / 1M tokens,$1.3 / hour
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,Hugging Face Inference Endpoint,$0.068 / 1M tokens,$0.6 / hour
+gemini-pro,Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input characters, $0.5 / 1M output characters (60 queries per minute are free)"
+chat-bison (PaLM 2),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
+chat-bison-32k (PaLM 2 32K),Google VertexAI,"$0.25 / 1M input tokens, $0.5 / 1M output tokens","$0.25 / 1M input tokens, $0.5 / 1M output tokens"
+01-ai Yi Chat (34B),Together AI,$0.8 / 1M tokens,$0.8 / 1M tokens
+Chronos Hermes (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Deepseek Coder Instruct (33B),Together AI,$0.8 / 1M tokens,$0.8 / 1M tokens
+Platypus2 Instruct (70B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+MythoMax-L2 (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Vicuna v1.5 (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Vicuna v1.5 (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Code Llama Instruct (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Code Llama Instruct (34B),Together AI,$0.8 / 1M tokens,$0.8 / 1M tokens
+Code Llama Instruct (70B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Code Llama Instruct (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+LLaMA-2 Chat (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+LLaMA-2 Chat (70B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+LLaMA-2 Chat (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Mistral (7B) Instruct,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Mistral (7B) Instruct v0.2 (Together AI),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Mixtral-8x7B Instruct (46.7B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Nous Capybara v1.9 (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Nous Hermes 2 - Mixtral 8x7B-DPO (46.7B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Nous Hermes 2 - Mixtral 8x7B-SFT (46.7B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Nous Hermes LLaMA-2 (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Nous Hermes Llama-2 (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Nous Hermes-2 Yi (34B),Together AI,$0.8 / 1M tokens,$0.8 / 1M tokens
+OpenChat 3.5 (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+OpenOrca Mistral (7B) 8K,Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Qwen-Chat (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Qwen 1.5 Chat (0.5B),Together AI,$0.1 / 1M tokens,$0.1 / 1M tokens
+Qwen 1.5 Chat (1.8B),Together AI,$0.1 / 1M tokens,$0.1 / 1M tokens
+Qwen 1.5 Chat (4B),Together AI,$0.1 / 1M tokens,$0.1 / 1M tokens
+Qwen 1.5 Chat (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Qwen 1.5 Chat (14B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Qwen 1.5 Chat (72B),Together AI,$0.9 / 1M tokens,$0.9 / 1M tokens
+Snorkel Mistral PairRM DPO (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Alpaca (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+OpenHermes-2-Mistral (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+OpenHermes-2.5-Mistral (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+Falcon Instruct (40B),Together AI,$0.8 / 1M tokens,$0.8 / 1M tokens
+Falcon Instruct (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+LLaMA-2-7B-32K-Instruct (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+RedPajama-INCITE Chat (3B),Together AI,$0.1 / 1M tokens,$0.1 / 1M tokens
+RedPajama-INCITE Chat (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+StripedHyena Nous (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+ReMM SLERP L2 (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Toppy M (7B),Together AI,$0.2 / 1M tokens,$0.2 / 1M tokens
+WizardLM v1.2 (13B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens
+Upstage SOLAR Instruct v1 (11B),Together AI,$0.3 / 1M tokens,$0.3 / 1M tokens

data/summary.csv

@@ -0,0 +1,95 @@
+model,language,template_name,execution_time,characters_count,words_count,chunk_sizes_mean,chunk_sizes_min,chunk_sizes_max,chunk_generation_times_mean,chunk_generation_times_min,chunk_generation_times_max
+01-ai Yi Chat (34B),english,json,3.740751821820329,339.4756097560976,57.926829268292686,3.8364112458654906,1.0,14.0,0.028872832803768254,0.0004601478576660156,0.6119933128356934
+01-ai Yi Chat (34B),english,markdown,4.911168761369659,545.0121951219512,91.32926829268293,4.251426940639269,1.0,15.0,0.029400016551148402,0.000396728515625,0.6902577877044678
+01-ai Yi Chat (34B),japanese,json,13.06497314438891,388.7910447761194,11.104477611940299,1.1905393053016453,1.0,10.0,0.03675604747026014,0.00048804283142089844,0.6865222454071045
+01-ai Yi Chat (34B),japanese,markdown,20.104321274038863,587.4657534246576,17.876712328767123,1.151120654945645,1.0,13.0,0.036697812292211376,0.0004608631134033203,0.6928038597106934
+Chronos Hermes (13B),english,json,2.974407357828958,368.92857142857144,63.107142857142854,3.5136054421768708,1.0,15.0,0.02094727554559167,0.00027179718017578125,0.6065487861633301
+Chronos Hermes (13B),english,markdown,2.6719689165673604,361.8536585365854,61.97560975609756,3.6452088452088454,1.0,14.0,0.02059152469002351,0.0004761219024658203,0.28253698348999023
+Chronos Hermes (13B),japanese,json,15.704193201558343,687.3620689655172,23.344827586206897,1.1621337997376475,1.0,14.0,0.023474197327925606,0.0003821849822998047,0.3109428882598877
+Chronos Hermes (13B),japanese,markdown,11.713542679945627,1075.8,142.26666666666668,2.5384615384615383,1.0,16.0,0.023924544976345302,0.0004260540008544922,0.34424543380737305
+Falcon Instruct (7B),english,json,1.9191903869311016,82.66666666666667,15.766666666666667,4.235695986336465,1.0,13.0,0.012606756387868567,0.0004930496215820312,0.08649563789367676
+Falcon Instruct (7B),english,markdown,1.8954222866746246,89.24590163934427,15.245901639344263,4.60965283657917,1.0,13.0,0.012657315119356548,0.0003917217254638672,0.09382963180541992
+Falcon Instruct (7B),japanese,json,0.9907723726370395,35.37179487179487,7.051282051282051,3.6983914209115283,1.0,10.0,0.013643914509075258,0.00037479400634765625,0.0781710147857666
+Falcon Instruct (7B),japanese,markdown,0.6273794174194336,8.873417721518987,2.088607594936709,3.3864734299516908,1.0,13.0,0.010549438172492428,0.000865936279296875,0.034580230712890625
+LLaMA-2 Chat (7B),english,json,2.602991385893388,348.05194805194805,60.311688311688314,3.5179837227618798,1.0,14.0,0.007298828235414905,0.0002570152282714844,0.12662744522094727
+LLaMA-2 Chat (7B),english,markdown,2.678575509124332,402.0416666666667,69.375,3.747669601242879,1.0,16.0,0.0077138773707266855,0.00035452842712402344,0.2196364402770996
+LLaMA-2 Chat (7B),japanese,json,4.055958332334246,471.92857142857144,53.114285714285714,1.9620478707608244,1.0,14.0,0.009438785980169328,0.00035572052001953125,0.1930844783782959
+LLaMA-2 Chat (7B),japanese,markdown,4.446276148704634,717.1232876712329,67.6986301369863,1.8702440070022508,1.0,16.0,0.008583835081750738,0.0003447532653808594,0.2040574550628662
+Mistral (7B) Instruct v0.2 (Together AI),english,json,2.0136800810822053,344.508547008547,56.93589743589744,3.8069040423120515,1.0,15.0,0.00794907963001192,0.0002071857452392578,0.30161213874816895
+Mistral (7B) Instruct v0.2 (Together AI),english,markdown,2.5499956497567235,452.242774566474,73.121387283237,3.7100720789074355,1.0,15.0,0.007905251983898002,0.0002067089080810547,0.4635443687438965
+Mistral (7B) Instruct v0.2 (Together AI),japanese,json,3.768558966402976,271.26490066225165,5.172185430463577,1.0484271417236173,1.0,13.0,0.009928898496573373,0.00023865699768066406,2.616291046142578
+Mistral (7B) Instruct v0.2 (Together AI),japanese,markdown,7.687208134916764,1178.012658227848,138.30379746835442,1.9892057113543091,1.0,109.0,0.010960863498020433,0.00021409988403320312,8.214757204055786
+Mistral-7B-Instruct-v0.2,english,json,4.278648148264204,342.8,55.15,4.1311870534561415,1.0,15.0,0.040544769239987996,0.00024437904357910156,60.7178213596344
+Mistral-7B-Instruct-v0.2,english,markdown,3.7188014924526214,366.85,57.1375,4.239202657807309,1.0,16.0,0.03371991898375372,0.020945072174072266,0.25538039207458496
+Mistral-7B-Instruct-v0.2,japanese,json,2.413569121701377,52.07857142857143,3.414285714285714,1.2991803278688525,1.0,11.0,0.039267738759305254,0.00047779083251953125,3.370748996734619
+Mistral-7B-Instruct-v0.2,japanese,markdown,2.323612022399902,42.25,1.5833333333333333,1.1069868995633187,1.0,6.0,0.03958142719935121,0.0004932880401611328,0.2205650806427002
+Mixtral-8x7B-Instruct-v0.1,english,json,3.665129848137623,403.2542016806723,62.4390756302521,4.138125060633172,1.0,15.0,0.01437194579115682,0.0002009868621826172,3.4127349853515625
+Mixtral-8x7B-Instruct-v0.1,english,markdown,5.775103669417532,522.9172932330827,83.45864661654136,4.350284606242572,1.0,15.0,0.015317759848210679,0.00020313262939453125,1.7103569507598877
+Mixtral-8x7B-Instruct-v0.1,japanese,json,7.287098480350196,556.9218106995885,63.477366255144034,2.2494971825601304,1.0,109.0,0.02290278010061531,0.00020551681518554688,76.45081758499146
+Mixtral-8x7B-Instruct-v0.1,japanese,markdown,9.511569020294008,432.14285714285717,7.2976190476190474,1.0973066110456153,1.0,12.0,0.017938276822029118,0.00019669532775878906,3.0317282676696777
+OpenHermes-2.5-Mistral (7B),english,json,2.2801950351301445,356.66265060240966,58.87951807228916,3.619833700171191,1.0,15.0,0.009280798020646228,0.0003464221954345703,0.34207844734191895
+OpenHermes-2.5-Mistral (7B),english,markdown,2.5686250015308985,381.69736842105266,63.828947368421055,3.9398343066684776,1.0,15.0,0.009823806159897504,0.00034999847412109375,0.32857298851013184
+OpenHermes-2.5-Mistral (7B),japanese,json,2.385033627835716,143.78048780487805,4.060975609756097,1.0738682940158484,1.0,8.0,0.013302163463633877,0.0003523826599121094,0.34746313095092773
+OpenHermes-2.5-Mistral (7B),japanese,markdown,4.09419917478794,346.7682926829268,4.865853658536586,1.0120297540662704,1.0,9.0,0.010479846772686987,0.0003566741943359375,0.6548421382904053
+Qwen 1.5 Chat (7B),english,json,2.176741310312778,316.40506329113924,52.949367088607595,3.865161589608783,1.0,15.0,0.0067587282957351615,0.00036597251892089844,0.13403892517089844
+Qwen 1.5 Chat (7B),english,markdown,2.384753034084658,433.1898734177215,71.48101265822785,4.243799603174603,1.0,15.0,0.00671186927883398,0.00036978721618652344,0.14255690574645996
+Qwen 1.5 Chat (7B),japanese,json,2.2887367900413804,207.50632911392404,4.139240506329114,1.4821880650994574,1.0,17.0,0.007465819402801624,0.00038743019104003906,0.21312594413757324
+Qwen 1.5 Chat (7B),japanese,markdown,2.041359633575251,220.80246913580248,1.876543209876543,1.465142950765954,1.0,17.0,0.007345383120587039,0.00035572052001953125,0.13096904754638672
+RedPajama-INCITE Chat (7B),english,json,1.9923779017785017,56.0,5.514705882352941,2.9337442218798153,1.0,15.0,0.02580270341070848,0.00044846534729003906,0.131911039352417
+RedPajama-INCITE Chat (7B),english,markdown,2.0556714431099268,55.04347826086956,5.217391304347826,2.928296067848882,1.0,70.0,0.02626570611158149,0.0004153251647949219,0.939018964767456
+RedPajama-INCITE Chat (7B),japanese,json,1.801509298929354,53.58536585365854,5.853658536585366,2.816666666666667,1.0,13.0,0.02727127411426642,0.0003719329833984375,0.1054222583770752
+RedPajama-INCITE Chat (7B),japanese,markdown,1.896126465099614,77.59756097560975,4.439024390243903,4.019583070120025,1.0,70.0,0.02732501960749575,0.0007987022399902344,0.4024965763092041
+Snorkel Mistral PairRM DPO (7B),english,json,3.095366932551066,949.9333333333333,150.72,3.8147890340544013,1.0,15.0,0.007307618875579115,0.00022792816162109375,0.1967298984527588
+Snorkel Mistral PairRM DPO (7B),english,markdown,2.583486044406891,507.9625,78.75,4.004829013501528,1.0,15.0,0.00809382158769837,0.0003504753112792969,0.27209019660949707
+Snorkel Mistral PairRM DPO (7B),japanese,json,7.790234556680994,1704.5189873417721,208.60759493670886,1.981328075570531,1.0,15.0,0.008026355929728519,0.0003476142883300781,0.49362945556640625
+Snorkel Mistral PairRM DPO (7B),japanese,markdown,89.00719798347096,17120.925925925927,504.25925925925924,1.0784030121985644,1.0,14.0,0.005559155525883009,0.00032520294189453125,1.1380209922790527
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,json,1.9458302193674548,366.25,61.58965517241379,4.169774654522613,1.0,13.0,0.012296600931852907,0.00024318695068359375,0.17363858222961426
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,english,markdown,1.6830505162477494,282.55,47.1125,3.9572829131652663,1.0,14.0,0.012217252319600401,0.0003170967102050781,0.15227961540222168
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,json,1.4167885013989039,72.81428571428572,8.8,2.0274463007159906,1.0,13.0,0.014790566363975749,0.00031280517578125,0.3310587406158447
+TinyLlama/TinyLlama-1.1B-Chat-v1.0,japanese,markdown,1.336316986878713,89.78333333333333,12.716666666666667,2.432054176072235,1.0,13.0,0.013910464400900675,0.0005300045013427734,0.176621675491333
+Vicuna v1.5 (7B),english,json,2.851835834980011,372.4375,63.9875,3.508596325953839,1.0,15.0,0.011230722554045582,0.0003688335418701172,0.11755537986755371
+Vicuna v1.5 (7B),english,markdown,2.0539027697000747,382.4230769230769,64.55128205128206,3.7235051803769816,1.0,15.0,0.011191664883712751,0.00037550926208496094,0.1981511116027832
+Vicuna v1.5 (7B),japanese,json,4.896489579230547,304.5,16.9375,1.2847254268574066,1.0,15.0,0.015209688069330176,0.0003762245178222656,0.319460391998291
+Vicuna v1.5 (7B),japanese,markdown,5.635414824556949,301.55223880597015,3.2686567164179103,1.034511008704557,1.0,11.0,0.015171443860590671,0.0003802776336669922,0.19443845748901367
+WizardLM v1.2 (13B),english,json,3.101118081236539,280.52054794520546,48.0958904109589,3.2891101831031158,1.0,15.0,0.021355988586442935,0.00036406517028808594,0.4714198112487793
+WizardLM v1.2 (13B),english,markdown,3.5972929232650332,357.5138888888889,61.013888888888886,3.8293662600416543,1.0,14.0,0.02164479360378984,0.0005571842193603516,0.5012450218200684
+WizardLM v1.2 (13B),japanese,json,6.448913824267503,213.4878048780488,4.7682926829268295,1.0483262470806636,1.0,7.0,0.027856862946922703,0.00034737586975097656,0.5480227470397949
+WizardLM v1.2 (13B),japanese,markdown,8.740011346049425,292.6341463414634,7.634146341463414,1.0085319211532804,1.0,9.0,0.02801978238888618,0.0004551410675048828,0.8807919025421143
+chat-bison (PaLM 2),english,json,2.5406186148524283,381.7267857142857,60.88125,78.31727422604872,1.0,142.0,0.3495202942369709,0.0003688335418701172,7.334931373596191
+chat-bison (PaLM 2),english,markdown,2.9635989278554917,490.4875,78.5375,84.20386266094421,1.0,138.0,0.3641034907025841,0.0004639625549316406,4.600815534591675
+chat-bison (PaLM 2),japanese,json,3.0616454005241396,232.68,10.48,34.21764705882353,3.0,124.0,0.326771635167739,0.0005857944488525391,2.7724690437316895
+chat-bison (PaLM 2),japanese,markdown,4.669546687602997,431.725,13.85,31.74448529411765,2.0,124.0,0.27883222348549785,0.0005550384521484375,7.114030122756958
+chat-bison-32k (PaLM 2 32K),english,json,7.142214598400252,335.47232142857143,53.799107142857146,77.18344289235826,1.0,133.0,1.4512595853652734,0.00041985511779785156,101.5493803024292
+chat-bison-32k (PaLM 2 32K),english,markdown,3.44630788564682,429.95,69.35,82.6826923076923,1.0,133.0,0.5048719226167753,0.0005412101745605469,18.035786390304565
+chat-bison-32k (PaLM 2 32K),japanese,json,5.072022783756256,272.85,10.35,34.320754716981135,4.0,124.0,0.5336985453119818,0.0005052089691162109,52.47814655303955
+chat-bison-32k (PaLM 2 32K),japanese,markdown,6.1971998631954195,446.45,21.4,32.118705035971225,5.0,124.0,0.3856057263106751,0.0005400180816650391,7.165225505828857
+gemini-pro,english,json,2.7415290858475547,388.5206321334504,62.01755926251097,204.58853444290338,1.0,988.0,1.4202116817663073,0.0004703998565673828,13.73231840133667
+gemini-pro,english,markdown,3.0089986324310303,487.1458333333333,80.67708333333333,183.39607843137256,12.0,472.0,1.1130499073103362,0.0006177425384521484,4.875006198883057
+gemini-pro,japanese,json,4.5300196591070145,276.6694915254237,6.983050847457627,91.1927374301676,4.0,333.0,1.4752027082709627,0.0006062984466552734,8.291066646575928
+gemini-pro,japanese,markdown,6.166934008781727,422.7307692307692,9.096153846153847,90.83471074380165,9.0,266.0,1.3123543331445742,0.0006639957427978516,13.754565954208374
+gpt-3.5-turbo,english,call,2.908525016600803,456.85911016949154,72.26059322033899,,,,,,
+gpt-3.5-turbo,english,json,2.566863525494622,398.56960556844547,65.32366589327147,4.806746320443226,1.0,16.0,0.015585779219764062,0.0005395412445068359,1.4500916004180908
+gpt-3.5-turbo,english,markdown,2.3388922008184285,506.90384615384613,81.99038461538461,5.050584403142365,1.0,16.0,0.01715806382989308,0.0005617141723632812,1.127073049545288
+gpt-3.5-turbo,japanese,call,7.255881716940138,382.95,6.351111111111111,,,,,,
+gpt-3.5-turbo,japanese,json,5.226112197780009,237.1069182389937,3.559748427672956,1.1291481969569905,1.0,7.0,0.02129129201220198,0.0005452632904052734,13.75982117652893
+gpt-3.5-turbo,japanese,markdown,7.29901300883684,358.72131147540983,6.065573770491803,1.1894977168949772,1.0,15.0,0.021750167313542772,0.0005612373352050781,13.71114993095398
+gpt-4,english,call,9.15708539162022,374.16815144766144,59.661469933184854,,,,,,
+gpt-4,english,json,7.963626848197565,383.3707317073171,61.69512195121951,4.7834568389659005,1.0,16.0,0.08687007305138472,0.000537872314453125,3.531284809112549
+gpt-4,english,markdown,10.02310108244419,428.35,67.825,4.990243192078054,1.0,16.0,0.10377185442257747,0.0005447864532470703,2.087376117706299
+gpt-4,japanese,call,24.259297592319243,214.9076023391813,1.1695906432748537,,,,,,
+gpt-4,japanese,json,22.703774025460252,211.63025210084032,4.168067226890757,1.1685226429101707,1.0,7.0,0.11835955503744466,0.0005507469177246094,3.5608017444610596
+gpt-4,japanese,markdown,31.99519391655922,275.1,2.925,1.1327980234712787,1.0,9.0,0.12606545905407157,0.0005619525909423828,2.9794347286224365
+gpt-4-turbo,english,call,7.319288367695279,372.86,58.532222222222224,,,,,,
+gpt-4-turbo,english,json,6.457074524425879,407.0243902439024,64.18414634146342,4.735998183701559,1.0,17.0,0.06474667943090798,0.0006806850433349609,5.431884765625
+gpt-4-turbo,english,markdown,8.325546002388,462.1125,71.925,4.984360253471754,1.0,16.0,0.07950969425127868,0.0006999969482421875,2.0926766395568848
+gpt-4-turbo,japanese,call,32.67405146055443,372.7860465116279,24.886046511627907,,,,,,
+gpt-4-turbo,japanese,json,19.359180968999862,250.44166666666666,6.183333333333334,1.184261339007763,1.0,8.0,0.08582140138847597,0.0006830692291259766,13.252992153167725
+gpt-4-turbo,japanese,markdown,20.977174520492554,283.55,4.45,1.1460038395473375,1.0,10.0,0.07991695286011809,0.0007085800170898438,2.3845231533050537
+llama-2-70b-chat,english,json,2.917410132025375,355.7463556851312,54.861030126336246,4.02267032967033,1.0,15.0,0.019103390617685005,0.0001990795135498047,3.8762006759643555
+llama-2-70b-chat,english,markdown,4.99397261045417,827.5714285714286,125.95918367346938,4.177931176591799,1.0,16.0,0.018161741699259625,0.0002124309539794922,0.7439091205596924
+llama-2-70b-chat,japanese,json,5.265992468681889,369.3768115942029,42.56521739130435,2.3362207250561435,1.0,15.0,0.023359512199899433,0.00020837783813476562,0.9314718246459961
+llama-2-70b-chat,japanese,markdown,11.084085723449444,1379.6896551724137,158.01724137931035,2.2644745033674796,1.0,16.0,0.01566780037868879,0.0002117156982421875,76.44498753547668
+zephyr-7b-beta,english,json,4.252429554081262,376.7899686520376,61.78369905956113,4.26030553291036,1.0,14.0,0.03764735399563352,0.0003070831298828125,1.0504331588745117
+zephyr-7b-beta,english,markdown,4.788659170695714,376.10714285714283,61.15,4.393408427200668,1.0,15.0,0.04506704832147846,0.00040459632873535156,75.15552616119385
+zephyr-7b-beta,japanese,json,2.6574003274624163,44.75,3.6923076923076925,1.152208358090711,1.0,7.0,0.044344271615274496,0.018090486526489258,1.1675689220428467
+zephyr-7b-beta,japanese,markdown,2.6084938406944276,40.0,2.39,1.0282776349614395,1.0,4.0,0.042658403845250145,0.02583909034729004,1.2674541473388672

pipeline/config.py

@@ -36,7 +36,7 @@ class QueriesConfig(Config):
 class MeasurementsConfig(Config):
     mock: bool = False
     remove_old_measurements: bool = False
-    small_dataset: bool = True
+    small_dataset: bool = False
 
 
 class QueriesDatasetConfig(Config):

pipeline/models.py

@@ -1,5 +1,6 @@
 import os
 from dataclasses import dataclass
+import re
 
 
 @dataclass(unsafe_hash=True)
@@ -525,6 +526,27 @@ MISSING_MEASUREMENTS = ['Mistral-7B-Instruct-v0.2',
  'WizardLM v1.2 (13B)',
  'Upstage SOLAR Instruct v1 (11B)']
 
+def together_ai_price(billions_of_parameters):
+    if billions_of_parameters <= 4:
+        return 0.1
+    elif 4.1 <= billions_of_parameters <= 8:
+        return 0.2
+    elif 8.1 <= billions_of_parameters <= 21:
+        return 0.3
+    elif 21.1 <= billions_of_parameters <= 41:
+        return 0.8
+    elif 41 < billions_of_parameters: # <= 70:
+        return 0.9
+
 for model in MODELS:
     if model.model_display_name in MISSING_MEASUREMENTS:
         model.selected = True
+    
+    if model.provider == "Together AI":
+        pattern = r"\((\d+(\.\d+)?)B\)"
+        match = re.search(pattern, model.model_display_name)
+        if match:
+            number_in_parenthesis = float(match.group(1))
+            model.size_billion_parameters = number_in_parenthesis
+            model.cost_per_million_tokens = together_ai_price(model.size_billion_parameters)
+            model.__post_init__()