Update app.py

app.py

@@ -74,7 +74,7 @@ peft_model_id = "imdeadinside410/Llama2-Syllabus"
 config = PeftConfig.from_pretrained(peft_model_id)
 
 model = AutoModelForCausalLM.from_pretrained(
-    config.base_model_name_or_path, return_dict=True)
+    config.base_model_name_or_path, return_dict=True, load_in_4bit=True, device_map="auto")
 
 tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
 
@@ -84,9 +84,7 @@ model = PeftModel.from_pretrained(model, peft_model_id)
 
 pipe = pipeline(task="text-generation",
                 model=model,
-                tokenizer=tokenizer,
-                device=-1,  # -1 indicates CPU
-                max_length=300)
+                tokenizer=tokenizer, max_length=300)
 
 
 
@@ -158,14 +156,14 @@ def main():
     conversation = st.session_state.get("conversation", [])
 
     query = st.text_input("Please input your question here:", key="user_input")
-    result = pipe(f"<s>[INST] {query} [/INST]")
+    result = pipe(f"<s>[INST] {prompt} [/INST]")
     if st.button("Get Answer"):
         if query:
             # Display the processing message
             with st.spinner("Processing your question..."):
                 conversation.append({"role": "user", "message": query})
                 # Call your QA function
-                answer = result
+                answer = result[0]['generated_text'].split("[/INST]")[1]
                 conversation.append({"role": "bot", "message": answer})
                 st.session_state.conversation = conversation
         else: