update

app.py

@@ -1,77 +1,74 @@
-import gradio as gr
 import spaces
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
-import threading
+import os
+import gradio as gr
+import sentencepiece
+
+
+os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:120'
+model_id = "thesven/Llama3-8B-SFT-code_bagel-bnb-4bit"
+tokenizer_path = "./"
+
+DESCRIPTION = """
+# thesven/Llama3-8B-SFT-code_bagel-bnb-4bit
+"""
 
-model_to_use = "thesven/Llama3-8B-SFT-code_bagel-bnb-4bit"
+tokenizer = AutoTokenizer.from_pretrained(model_id, device_map="auto", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16, trust_remote_code=True)
 
-# Initialize global variables for the tokenizer and model
-tokenizer = None
-model = None
+def format_prompt(user_message, system_message="You are an expert developer in all programming languages.  Help me with my code. Answer any questions I have with code examples."):
+    prompt = f"<|im_start|>assistant\n{system_message}<|im_end|>\n<|im_start|>\nuser\n{user_message}<|im_end|>\nassistant\n"
+    return prompt
 
 @spaces.GPU
-def load_model():
-    global tokenizer, model
-    model_name_or_path = model_to_use
+def predict(message, system_message, max_new_tokens=600, temperature=3.5, top_p=0.9, top_k=40, do_sample=False):
+    formatted_prompt = format_prompt(message, system_message)
 
-    # BitsAndBytesConfig for loading the model in 4-bit precision
-    bnb_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_compute_dtype="bfloat16",
-    )
+    input_ids = tokenizer.encode(formatted_prompt, return_tensors='pt')
+    input_ids = input_ids.to(model.device)
 
-    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name_or_path,
-        device_map="auto",
-        trust_remote_code=True,
-        quantization_config=bnb_config
+    response_ids = model.generate(
+        input_ids,
+        max_length=max_new_tokens + input_ids.shape[1],
+        temperature=temperature,  
+        top_p=top_p,              
+        top_k=top_k,              
+        no_repeat_ngram_size=9,
+        pad_token_id=tokenizer.eos_token_id,
+        do_sample=do_sample
     )
-    model.pad_token_id = model.config.eos_token_id
-
-    return "Model loaded and ready!"
 
-def send_message(message, history):
-    global tokenizer, model
-    if tokenizer is None or model is None:
-        return history  # Return the existing history if the model is not loaded
+    response = tokenizer.decode(response_ids[:, input_ids.shape[-1]:][0], skip_special_tokens=True)
+    truncate_str = "<|im_end|>"
+    if truncate_str and truncate_str in response:
+        response = response.split(truncate_str)[0]
 
-    # Add the user's message to the history
-    history.append(("User", message))
-    
-    # Generate the model's response
-    input_text = " ".join([msg for _, msg in history])
-    input_ids = tokenizer(input_text, return_tensors='pt').input_ids.cuda()
-    output = model.generate(inputs=input_ids, max_new_tokens=50)
-    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
-    
-    # Add the model's response to the history
-    history.append(("Bot", generated_text))
+    return [("bot", response)]    
     
-    return history
+with gr.Blocks() as demo:
+    gr.Markdown(DESCRIPTION)
+    with gr.Group():
+        system_prompt = gr.Textbox(placeholder='Provide a System Prompt In The First Person', label='System Prompt', lines=2, value="ou are an expert developer in all programming languages.  Help me with my code. Answer any questions I have with code examples.")
 
-def initialize():
-    # Function to run the model loading in a separate thread
-    threading.Thread(target=load_model).start()
+    with gr.Group():
+        chatbot = gr.Chatbot(label='thesven/Llama3-8B-SFT-code_bagel-bnb-4bit')
 
-with gr.Blocks() as demo:
-    gr.Markdown("# Chat with the Model")
-    
-    status_text = gr.Textbox(label="Status", value="Loading model, please wait...")
-    send_button = gr.Button("Send", interactive=False)  # Disable the send button initially
-    
-    chatbot = gr.Chatbot()
-    message = gr.Textbox(label="Your Message")
-    
-    def enable_send_button():
-        send_button.interactive = True
-        status_text.value = "Model loaded and ready!"
+    with gr.Group():
+        textbox = gr.Textbox(placeholder='Your Message Here', label='Your Message', lines=2)
+        submit_button = gr.Button('Submit', variant='primary')
 
-    demo.load(_js="initialize(); enable_send_button();")
-    send_button.click(send_message, inputs=[message, chatbot], outputs=chatbot)
-    
-initialize()  # Start model initialization on app load
+    with gr.Accordion(label='Advanced options', open=False):
+        max_new_tokens = gr.Slider(label='Max New Tokens', minimum=1, maximum=55000, step=1, value=4056)
+        temperature = gr.Slider(label='Temperature', minimum=0.1, maximum=4.0, step=0.1, value=1.2)
+        top_p = gr.Slider(label='Top-P (nucleus sampling)', minimum=0.05, maximum=1.0, step=0.05, value=0.9)
+        top_k = gr.Slider(label='Top-K', minimum=1, maximum=1000, step=1, value=40)
+        do_sample_checkbox = gr.Checkbox(label='Disable for faster inference', value=True)
+
+    submit_button.click(
+        fn=predict,
+        inputs=[textbox, system_prompt, max_new_tokens, temperature, top_p, top_k, do_sample_checkbox],
+        outputs=chatbot
+    )
 
 demo.launch()

requirements.txt

@@ -1,4 +1,5 @@
 accelerate
 bitsandbytes
 transformers
-spaces
+spaces
+sentencepiece