Add router app

.streamlit/config.toml

@@ -0,0 +1,4 @@
+[theme]
+base="light"
+primaryColor="#00b828"
+font="serif"

README.md

@@ -1,8 +1,8 @@
 ---
 title: Llmrouter
-emoji: 🏢
-colorFrom: green
-colorTo: green
+emoji: 📉
+colorFrom: purple
+colorTo: purple
 sdk: streamlit
 sdk_version: 1.32.2
 app_file: app.py

app.py

@@ -0,0 +1,150 @@
+from openai import OpenAI
+import streamlit as st
+import numpy as np
+from PIL import Image
+from time import perf_counter
+
+# Page configuration, header section, and images
+st.set_page_config(
+    page_title= "Unify Router Demo",
+    page_icon="./assets/unify_spiral.png",
+    layout = "wide",
+    initial_sidebar_state="collapsed"
+)
+
+Header = st.columns(3)
+with Header[1]:
+    st.image(
+        "./assets/unify_logo.png",
+        use_column_width="auto",
+        caption="Route your prompt to the best LLM"
+    )
+st.write("Chat with the Unify LLM router! Send your prompt to the best LLM endpoint, optimizing for the metric of your choice. For any given model, the router searches across endpoints from different model endpoint providers to find the one endpoint that will provide the best performance for the target metric, for each prompt")
+
+st.info(
+    body="This demo is only a preview of the router's functionalities. Check out our [Chat UI](https://unify.ai/router) for the full experience, including more endpoints, and extra customization!",
+    icon="ℹ️"
+)
+
+router_avatar = Image.open('./assets/unify_spiral.png')
+
+# Parameter choices
+strategies = {
+    '🏃 fastest': "tks-per-sec",
+    '⌛ most responsive': "ttft",
+    "💵 cheapest": "input-cost",
+}
+models = {
+    '🦙 Llama2 70B Chat': "llama-2-70b-chat",
+    '🕸️ Mixtral 8x7B Instruct': "mixtral-8x7b-instruct-v0.1",
+    '👨‍💻 Deepseek Coder 33B Instruct': "deepseek-coder-33b-instruct",
+}
+
+# Body
+Parameters_Col, Chat_Col = st.columns([1,3])
+
+with Parameters_Col:
+
+    strategy = st.selectbox(
+        label = 'I want the', 
+        options = tuple(strategies.keys()), 
+        help="Choose the metric to optimize the routing for. \
+            Fastest picks the endpoint with the highest output tokens per seconds. \
+            Most responsive picks the endpoint with the smallest time to complete the request. \
+            Cheapest picks the endpoint with the lowest output tokens cost",
+    )
+    model = st.selectbox(
+        label = 'endpoint for',
+        options = tuple(models.keys()), 
+        help="Select a model to optimize for. The same model can be offered by different model endpoint providers. The router lets you find the optimal endpoint for your chosen model, target metric, and input prompt",
+    )
+    with st.expander("Advanced Inputs"):
+        max_tokens = st.slider(
+            label = "Maximum Number Of Tokens",
+            min_value=10,
+            max_value=500,
+            value=100,
+            step=20,
+            help = "The maximum number of tokens that can be generated."
+        )
+        temperature = st.slider(
+            label = "Temperature",
+            min_value=0.0,
+            max_value=1.,
+            value=0.5,
+            step=0.5,
+            help = "The model's output randomness. Higher values give more random outputs."
+        )
+
+with Chat_Col:
+
+    st.write("Chat with Router")
+    # Initializing empty chat space
+    if "messages" not in st.session_state:
+        st.session_state.messages = []
+    msgs = st.container(height=300)
+
+    # Writing conversation history
+    for msg in st.session_state.messages:
+        msgs.chat_message(msg["role"]).write(msg["content"])
+
+    # Preparing client    
+    client = OpenAI(
+        base_url="https://api.unify.ai/v0/",
+        api_key=st.secrets["UNIFY_API"]
+    )
+
+    # Processing prompt box input
+    if prompt := st.chat_input("Enter your prompt.."):
+
+        # Displaying user prompt and saving in message states
+        msgs.chat_message("user").write(prompt)
+        st.session_state.messages.append({"role": "user", "content": prompt})
+    
+        # Sending prompt to model endpoint
+        start = perf_counter()
+        stream = client.chat.completions.create(
+            model="@".join([
+                    models[model],
+                    strategies[strategy]
+                ]),
+            messages=[
+                {"role": m["role"], "content": m["content"]}
+                for m in st.session_state.messages
+            ],
+            stream=True,
+            max_tokens=max_tokens,
+            temperature=temperature
+        )
+        time_to_completion = round(perf_counter() - start, 2)
+        # Displaying output, metrics, and saving output in message states
+        with msgs.chat_message("assistant", avatar=np.array(router_avatar)):
+
+            # Writing answer progressively
+            chunks = [chunk for chunk in stream]
+            st.write_stream(chunks)
+
+            # Computing metrics
+            last_chunk = chunks[-1]
+            cost = round(last_chunk.usage["cost"],6)
+            output_tokens = last_chunk.usage["completion_tokens"]
+            tokens_per_second = round(output_tokens / time_to_completion, 2)
+            
+            # Displaying model, provider, and metrics
+            provider = " ".join(chunks[0].model.split("@")[-1].split("-")).title()
+            st.markdown(f"Model: **{model}**. Provider: **{provider}**")
+            st.markdown(
+                f"**{tokens_per_second}** Tokens Per Second - \
+                  **{time_to_completion}** Seconds to complete - \
+                  **{cost}** $"
+                )
+
+        # Saving output to message states
+        output_chunks = [chunk.choices[0].delta.content or "" for chunk in chunks]
+        response = "".join(output_chunks) 
+        st.session_state.messages.append({"role": "assistant", "content": response})
+
+    # Cancel / Stop button
+    if st.button("Clear Chat", key="clear"):
+        msgs.empty()
+        st.session_state.messages = []

requirements.txt

@@ -0,0 +1,4 @@
+streamlit==1.32.0
+openai
+numpy
+pillow