Update app.py

app.py

@@ -12,13 +12,17 @@ model_id = "eastwind/grok-1-hf-4bit"
 tokenizer_id = "Xenova/grok-1-tokenizer"
 # tokenizer_path = "./"
 # eos_token_id = 7
-
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
 DESCRIPTION = """
 # Welcome to Tonic's Grok-1
 """
 
 # tokenizer = AutoTokenizer.from_pretrained(model_id, device_map="auto", trust_remote_code=True)
-tokenizer = LlamaTokenizerFast.from_pretrained(tokenizer_id, device_map="cuda", trust_remote_code=True)
+tokenizer = LlamaTokenizerFast.from_pretrained(tokenizer_id, device_map="cuda", quantization_config = quantization_config, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True, device_map="cuda", trust_remote_code=True)
 
 def format_prompt(user_message, system_message="You are Grok-1, an AI language model created by Tonic-AI. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and follow ethical guidelines and promote positive behavior.\n\n"):