Update README.md

README.md

@@ -26,28 +26,47 @@ https://ai.meta.com/llama/
 
 ## How to use
 
+
+
 ```
 from huggingface_hub import notebook_login
 notebook_login()
 ```
 
-```python
-from peft import PeftModel, PeftConfig
-from transformers import AutoModelForCausalLM
-
-config = PeftConfig.from_pretrained("doshisha-nlp/llama-2-70b-chat-4bit-alpaca-japanese")
-model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-70b-chat-hf")
-model = PeftModel.from_pretrained(model, "doshisha-nlp/llama-2-70b-chat-4bit-alpaca-japanese")
 
+```python
+import torch
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+model_id = "meta-llama/Llama-2-70b-chat-hf"
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16,
+)
+
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map="auto")
+
+peft_name = "doshisha-nlp/llama-2-70b-chat-4bit-alpaca-japanese"
+model = PeftModel.from_pretrained(
+    model, 
+    peft_name, 
+    is_trainable=True
+)
 model.eval()
+
 device = "cuda:0"
-text = "#Q:日本一高い山は何ですか？ #A: "
+
+text = "# Q: 日本一高い山は何ですか？ # A: "
 inputs = tokenizer(text, return_tensors="pt").to(device)
 with torch.no_grad():
   outputs = model.generate(**inputs, max_new_tokens=100)
-
 print(tokenizer.decode(outputs[0], skip_special_tokens=True))
 
+
 ```
 ## Training procedure