Update app.py

app.py

@@ -1,43 +1,46 @@
-import torch
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
-
-model_name = "mistralai/Mistral-7B-Instruct-v0.1"
-
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-tokenizer.padding_side = "left"
-tokenizer.pad_token = tokenizer.eos_token
-
-# Load model in 4-bit quantization
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    device_map="auto",
-    load_in_4bit=True,
-    torch_dtype=torch.float16
-)
-
-# Create text generation pipeline
-chatbot = pipeline(
-    "text-generation", 
-    model=model, 
-    tokenizer=tokenizer, 
-    device=0, 
-    pad_token_id=tokenizer.eos_token_id
-)
-
-# Function for chatting
-def mistral_chat(user_input):
-    response = chatbot(user_input, max_new_tokens=200, temperature=0.7, do_sample=True)
-    return response[0]["generated_text"]
-
-# Gradio interface
-iface = gr.Interface(
-    fn=mistral_chat,
-    inputs="text",
-    outputs="text",
-    title="Mistral 7B Chatbot"
+from huggingface_hub import InferenceClient
+import os
+
+# Load HF Token from Environment Variables (Make sure to add it in Hugging Face Secrets)
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+# Use Mistral-7B instead of Zephyr
+client = InferenceClient("mistralai/Mistral-7B-Instruct-v0.1", token=HF_TOKEN)
+
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    messages = [{"role": "system", "content": system_message}]
+
+    for user_msg, bot_msg in history:
+        if user_msg:
+            messages.append({"role": "user", "content": user_msg})
+        if bot_msg:
+            messages.append({"role": "assistant", "content": bot_msg})
+
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+    
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=False,  # Change to True if streaming works
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        response += message.choices[0].delta.content
+        yield response
+
+# Gradio Chat Interface
+demo = gr.ChatInterface(
+    respond,
+    additional_inputs=[
+        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)"),
+    ],
 )
 
-iface.launch()
-
+if __name__ == "__main__":
+    demo.launch()