chore: use int4

app.py

@@ -36,7 +36,12 @@ except:
 
 MODEL_NAME = "hatakeyama-llm-team/Tanuki-8B-Instruct"
 
-quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_NAME, quantization_config=quantization_config, device_map="auto"
 )