try llama cpp

app.py

@@ -1,12 +1,7 @@
-import os
-os.environ["CUDA_VISIBLE_DEVICES"] = ""
-
 import gradio as gr
 
-#from unsloth import FastLanguageModel
-from peft import AutoPeftModelForCausalLM
-from transformers import TextIteratorStreamer, AutoTokenizer
-from threading import Thread
+from llama_cpp import Llama
+from llama_cpp.llama_chat_format import MoondreamChatHandler
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
@@ -17,7 +12,6 @@ class MyModel:
     def __init__(self):
         self.client = None
         self.current_model = ""
-        self.tokenizer = None
 
     def respond(
         self,
@@ -30,21 +24,18 @@ class MyModel:
         min_p,
     ):
         if model != self.current_model or self.current_model is None:
-            # client, tokenizer = FastLanguageModel.from_pretrained(
-            #     model_name = model,
-            #     max_seq_length = 2048,
-            #     dtype = None,
-            #     load_in_4bit = True,
-            # )
-            # FastLanguageModel.for_inference(client) # Enable native 2x faster inference
-            tokenizer = AutoTokenizer.from_pretrained(model)
-            client = AutoPeftModelForCausalLM.from_pretrained(model, load_in_4bit=True)
+            chat_handler = MoondreamChatHandler.from_pretrained(
+                repo_id="lab2-as/lora_model_gguf",
+            )
+            client = Llama.from_pretrained(
+                repo_id="lab2-as/lora_model_gguf",
+                chat_handler=chat_handler,
+                n_ctx=2048, # n_ctx should be increased to accommodate the image embedding
+            )
 
             self.client = client
-            self.tokenizer = tokenizer
             self.current_model = model
         
-        text_streamer = TextIteratorStreamer(self.tokenizer, skip_prompt = True)
 
         messages = [{"role": "system", "content": system_message}]
 
@@ -56,22 +47,19 @@ class MyModel:
 
         messages.append({"role": "user", "content": message})
 
-        inputs = self.tokenizer.apply_chat_template(
-            messages,
-            tokenize = True,
-            add_generation_prompt = True, # Must add for generation
-            return_tensors = "pt",
-        )
-        
-        generation_kwargs = dict(input_ids=inputs, streamer=text_streamer, max_new_tokens=max_tokens, use_cache=True, temperature=temperature, min_p=min_p)
-        thread = Thread(target=self.client.generate, kwargs=generation_kwargs)
-        thread.start()
-
         response = ""
 
-        for new_text in text_streamer:
-            response += new_text
-            yield response.strip("<|eot_id|>")
+        for message in self.client.create_chat_completion(
+                messages,
+                temperature=temperature,
+                top_p=min_p,
+                stream=True,
+                max_tokens=max_tokens
+            ):
+            delta = message["choices"][0]["delta"]
+            if "content" in delta:
+                response += delta["content"]
+                yield response
 
         # for message in client.chat_completion(
         #     messages,

requirements.txt

@@ -1,6 +1,2 @@
 huggingface_hub==0.25.2
-transformers>=4.45.1
-accelerate
-peft
-torch
-#https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_multi-backend-refactor/bitsandbytes-0.44.1.dev0-py3-none-manylinux_2_24_x86_64.whl
+llama-cpp-python