Update

README.md

@@ -15,38 +15,28 @@ language:
 LLMでの出力方法
 
 ```python
-from transformers import (
-    AutoModelForCausalLM,
-    AutoTokenizer,
-    BitsAndBytesConfig,
-)
+%%capture
+!pip install unsloth
+!pip uninstall unsloth -y && pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
+
+from unsloth import FastLanguageModel
 import torch
-from tqdm import tqdm
 import json
 
-HF_TOKEN = "HF_TOKEN"
 model_name = "tatsurou/llm-jp-3-13b-ft-20241123"
-
-# QLoRA config
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.bfloat16,
-    bnb_4bit_use_double_quant=False,
-)
-
-# Load model
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-    quantization_config=bnb_config,
-    device_map="auto",
-    token = HF_TOKEN
+max_seq_length = 2048
+dtype = None
+load_in_4bit = True
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = model_name,
+    max_seq_length = max_seq_length,
+    dtype = dtype,
+    load_in_4bit = load_in_4bit,
+    token = "HF token",
 )
+FastLanguageModel.for_inference(model)
 
-# Load tokenizer
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, token = HF_TOKEN)
-
-# データセットの読み込み。
 datasets = []
 with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
     item = ""
@@ -56,35 +46,27 @@ with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
       if item.endswith("}"):
         datasets.append(json.loads(item))
         item = ""
+
+from tqdm import tqdm
+
+# 推論
 results = []
-for data in tqdm(datasets):
+for dt in tqdm(data):
+  input = dt["input"]
 
-  input = data["input"]
+  prompt = f"""### 指示\n{input}\n### 回答\n"""
 
-  prompt = f"""### 指示
-  {input}
-  ### 回答：
-  """
+  inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
 
-  tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
-  with torch.no_grad():
-      outputs = model.generate(
-          tokenized_input,
-          max_new_tokens=300,
-          do_sample=False,
-          repetition_penalty=1.2
-      )[0]
-  output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
+  outputs = model.generate(**inputs, max_new_tokens = 512, use_cache = True, do_sample=False, repetition_penalty=1.2)
+  prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split('\n### 回答')[-1]
 
   results.append({"task_id": data["task_id"], "input": input, "output": output})
 
-import re
-model_name = re.sub(".*/", "", model_name)
-with open(f"./{model_name}-my-original-outputs.jsonl", 'w', encoding='utf-8') as f:
+with open(f"/content/{model_name}_output.jsonl", 'w', encoding='utf-8') as f:
     for result in results:
-        json.dump(result, f, ensure_ascii=False)  # ensure_ascii=False for handling non-ASCII characters
+        json.dump(result, f, ensure_ascii=False)
         f.write('\n')
-
 ```