Added code for inference using our model

.gitignore

@@ -0,0 +1 @@
+/venv

app.py

@@ -1,21 +1,15 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
+from unsloth import FastLanguageModel
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
 #client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-client = InferenceClient("halme/id2223_lora_model")
+#client = InferenceClient("halme/id2223_lora_model")
 
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
+def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p,):
     messages = [{"role": "system", "content": system_message}]
 
     for val in history:
@@ -28,17 +22,37 @@ def respond(
 
     response = ""
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
+    """ for message in client.chat_completion(messages, max_tokens=max_tokens, stream=True, temperature=temperature, top_p=top_p):
         token = message.choices[0].delta.content
 
         response += token
-        yield response
+        yield response """
+
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name = "halme/id2223_lora_model", # YOUR MODEL YOU USED FOR TRAINING
+        max_seq_length = max_tokens,
+        dtype = None,
+        load_in_4bit = True,
+    )
+
+    FastLanguageModel.for_inference(model) # Enable native 2x faster inference
+
+    """messages = [
+        {"role": "user", "content": "Continue the fibonnaci sequence: 1, 1, 2, 3, 5, 8,"},
+    ] """
+
+    inputs = tokenizer.apply_chat_template(
+        messages,
+        tokenize = True,
+        add_generation_prompt = True, # Must add for generation
+        return_tensors = "pt",
+    ).to("cuda")
+
+    from transformers import TextStreamer
+    text_streamer = TextStreamer(tokenizer, skip_prompt = True)
+    
+    yield model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 128,
+                    use_cache = True, temperature = 1.5, min_p = 0.1)
 
 
 """