fix: switch to 4-bit quantization

app.py

@@ -11,9 +11,10 @@ print("Loading tokenizer...")
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 tokenizer.pad_token = tokenizer.eos_token
 
-# Configure quantization
+# Configure quantization for 4-bit
 bnb_config = BitsAndBytesConfig(
-    load_in_8bit=True,
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float16
 )
 
 print("Loading base model...")