fix(model): improve 8-bit quantization configuration

Update 8-bit loading to use proper BitsAndBytesConfig with compute dtype
specification and fix torch_dtype handling when consumed by quantization.
Add bitsandbytes dependency and remove unused dependencies.

llava/model/builder.py

@@ -46,7 +46,10 @@ def load_pretrained_model(
         kwargs["device_map"] = {"": device}
 
     if load_8bit:
-        kwargs["load_in_8bit"] = True
+        kwargs["quantization_config"] = BitsAndBytesConfig(
+            load_in_8bit=True,
+            bnb_8bit_compute_dtype=torch.float16,
+        )
     elif load_4bit:
         kwargs["load_in_4bit"] = True
         kwargs["quantization_config"] = BitsAndBytesConfig(
@@ -158,4 +161,6 @@ def prepare_config_for_eval(config: PretrainedConfig, kwargs: dict):
     except AttributeError:
         raise ValueError(f"Invalid configuration! Cannot find vision_tower in config:\n{config}")
 
-    config.model_dtype = kwargs.pop("torch_dtype").__str__()
+    # Handle case where torch_dtype might be consumed by quantization config
+    torch_dtype = kwargs.pop("torch_dtype", torch.float16)
+    config.model_dtype = torch_dtype.__str__()

requirements.txt

@@ -4,14 +4,11 @@ hydra-core
 loguru
 Pillow
 pydub
-torch
-torchvision
 
 
 # Transformers and training utilities
 transformers==4.46.0
 pytorchvideo==0.1.5
-deepspeed==0.15.4
 accelerate==0.34.2
 numpy==1.26.4
 opencv-python-headless==4.8.0.76
@@ -27,8 +24,10 @@ jiwer
 einops
 wandb
 kaldiio
-peft==0.14.0
+peft
 
 # Compatibility fix
 protobuf==3.20.*
-triton==3.1.0
+triton==3.1.0
+
+bitsandbytes