Update app.py

app.py

@@ -22,7 +22,7 @@ from peft import PeftModel, PeftConfig
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 config = PeftConfig.from_pretrained("tmberooney/medstralchatbotgptq")
-model = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-Instruct-v0.2-DARE-GPTQ")
+model = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-Instruct-v0.2-DARE-GPTQ",load_in_4bit=True, torch_dtype=torch.float16, llm_int8_enable_fp32_cpu_offload=True)
 model = PeftModel.from_pretrained(model, "tmberooney/medstralchatbotgptq")
 tokenizer=AutoTokenizer.from_pretrained(config.base_model_name_or_path)
 device_map = {"transformer.word_embeddings": "cpu",