bug fix

app.py

@@ -2,13 +2,18 @@ import torch
 import gradio as gr
 from transformers import AutoTokenizer
 from auto_gptq import AutoGPTQForCausalLM
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Hindari beban CPU
 
 # Load model & tokenizer
 # model_name_or_path = "TheBloke/Llama-2-7B-Chat-GPTQ"
 print("PyTorch Version",torch.__version__)  # Versi PyTorch
 print("Is GPU Available",torch.cuda.is_available())  # Apakah GPU terdeteksi?
+print("CPU cores:", psutil.cpu_count())
+print("RAM (GB):", psutil.virtual_memory().total / (1024**3))
 
-model_name_or_path = "TheBloke/Llama-2-7B-Chat-GGUF"
+# model_name_or_path = "TheBloke/Llama-2-7B-Chat-GGUF"
+model_name_or_path = "TheBloke/Mistral-7B-v0.1-GPTQ"
 # tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
 tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
 
@@ -17,13 +22,13 @@ model = AutoGPTQForCausalLM.from_quantized(
     model_basename="model",
     # device_map="auto",  # Auto-detects GPU/CPU
     device_map="cpu",
-    torch_dtype=torch.float32,  # <-- Tambahkan ini
+    torch_dtype=torch.float32,  # Hindari float16 di CPU
     use_safetensors=True,
     trust_remote_code=True,
     use_triton=False,
-    inject_fused_attention=False,
+    inject_fused_attention=False, # Wajib untuk CPU
     inject_fused_mlp=False,
-    disable_exllama=True,
+    disable_exllama=True, # Wajib untuk CPU
     disable_exllamav2=True,
 )
 
@@ -56,10 +61,10 @@ def chat(user_input, chat_history):
     with torch.inference_mode():
         output_ids = model.generate(
             input_ids=input_ids, 
-            max_new_tokens=50,
+            max_new_tokens=32,
             use_cache=True,
             # do_sample=True,
-            do_sample=False,
+            do_sample=False, # Matikan sampling untuk percepat
             temperature=0.7,
             top_p=0.95
         )

requirements.txt

@@ -1,7 +1,8 @@
 torch>=2.0.0
 gradio>=3.0.0
 transformers>=4.30.0
-auto-gptq>=0.4.0 --extra-index-url https://huggingface.github.io/autogptq-index/whl/cpu/
+auto-gptq>=0.4.0 --extra-index-url https://huggingface.github.io/autogptq-index/whl/cpu
+# auto-gptq --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118
 ninja>=1.11.0
 accelerate>=0.20.0
 bitsandbytes>=0.40.0