Parker commited on
Commit
d33e32a
·
verified ·
1 Parent(s): 467e717

Add quantized VibeVoice 7B models (4-bit and 8-bit)

Browse files
.gitattributes CHANGED
@@ -1,35 +1,5 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
  *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
  *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
 
2
  *.bin filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  *.pt filter=lfs diff=lfs merge=lfs -text
4
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
5
+ *.wav filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
4bit/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # VibeVoice 7B - 4-bit Quantized
2
+
3
+ Optimized for RTX 3060/4060 and similar 12GB VRAM GPUs.
4
+
5
+ ## Specifications
6
+ - Quantization: 4-bit (nf4)
7
+ - Model size: 6.2 GB
8
+ - VRAM usage: ~8 GB
9
+ - Quality: Very good (minimal degradation)
10
+
11
+ ## Usage
12
+
13
+ ```python
14
+ from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
15
+ from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
16
+
17
+ model = VibeVoiceForConditionalGenerationInference.from_pretrained(
18
+ "Dannidee/VibeVoice7b-low-vram/4bit",
19
+ device_map='cuda',
20
+ torch_dtype=torch.bfloat16,
21
+ )
22
+ processor = VibeVoiceProcessor.from_pretrained("Dannidee/VibeVoice7b-low-vram/4bit")
23
+
24
+ # Generate speech
25
+ text = "Speaker 1: Hello! Speaker 2: Hi there!"
26
+ inputs = processor(
27
+ text=[text],
28
+ voice_samples=[["voice1.wav", "voice2.wav"]],
29
+ padding=True,
30
+ return_tensors="pt",
31
+ )
32
+
33
+ outputs = model.generate(**inputs)
34
+ processor.save_audio(outputs.speech_outputs[0], "output.wav")
35
+ ```
4bit/config.json ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "acoustic_vae_dim": 64,
3
+ "acoustic_tokenizer_config": {
4
+ "causal": true,
5
+ "channels": 1,
6
+ "conv_bias": true,
7
+ "conv_norm": "none",
8
+ "corpus_normalize": 0.0,
9
+ "decoder_depths": null,
10
+ "decoder_n_filters": 32,
11
+ "decoder_ratios": [
12
+ 8,
13
+ 5,
14
+ 5,
15
+ 4,
16
+ 2,
17
+ 2
18
+ ],
19
+ "disable_last_norm": true,
20
+ "encoder_depths": "3-3-3-3-3-3-8",
21
+ "encoder_n_filters": 32,
22
+ "encoder_ratios": [
23
+ 8,
24
+ 5,
25
+ 5,
26
+ 4,
27
+ 2,
28
+ 2
29
+ ],
30
+ "fix_std": 0.5,
31
+ "layer_scale_init_value": 1e-06,
32
+ "layernorm": "RMSNorm",
33
+ "layernorm_elementwise_affine": true,
34
+ "layernorm_eps": 1e-05,
35
+ "mixer_layer": "depthwise_conv",
36
+ "model_type": "vibevoice_acoustic_tokenizer",
37
+ "pad_mode": "constant",
38
+ "std_dist_type": "gaussian",
39
+ "vae_dim": 64,
40
+ "weight_init_value": 0.01
41
+ },
42
+ "architectures": [
43
+ "VibeVoiceForConditionalGeneration"
44
+ ],
45
+ "decoder_config": {
46
+ "attention_dropout": 0.0,
47
+ "hidden_act": "silu",
48
+ "hidden_size": 3584,
49
+ "initializer_range": 0.02,
50
+ "intermediate_size": 18944,
51
+ "max_position_embeddings": 32768,
52
+ "max_window_layers": 28,
53
+ "model_type": "qwen2",
54
+ "num_attention_heads": 28,
55
+ "num_hidden_layers": 28,
56
+ "num_key_value_heads": 4,
57
+ "rms_norm_eps": 1e-06,
58
+ "rope_scaling": null,
59
+ "rope_theta": 1000000.0,
60
+ "sliding_window": null,
61
+ "torch_dtype": "bfloat16",
62
+ "use_cache": true,
63
+ "use_mrope": false,
64
+ "use_sliding_window": false,
65
+ "vocab_size": 152064
66
+ },
67
+ "diffusion_head_config": {
68
+ "ddpm_batch_mul": 4,
69
+ "ddpm_beta_schedule": "cosine",
70
+ "ddpm_num_inference_steps": 20,
71
+ "ddpm_num_steps": 1000,
72
+ "diffusion_type": "ddpm",
73
+ "head_ffn_ratio": 3.0,
74
+ "head_layers": 4,
75
+ "hidden_size": 3584,
76
+ "latent_size": 64,
77
+ "model_type": "vibevoice_diffusion_head",
78
+ "prediction_type": "v_prediction",
79
+ "rms_norm_eps": 1e-05,
80
+ "speech_vae_dim": 64
81
+ },
82
+ "model_type": "vibevoice",
83
+ "semantic_tokenizer_config": {
84
+ "causal": true,
85
+ "channels": 1,
86
+ "conv_bias": true,
87
+ "conv_norm": "none",
88
+ "corpus_normalize": 0.0,
89
+ "disable_last_norm": true,
90
+ "encoder_depths": "3-3-3-3-3-3-8",
91
+ "encoder_n_filters": 32,
92
+ "encoder_ratios": [
93
+ 8,
94
+ 5,
95
+ 5,
96
+ 4,
97
+ 2,
98
+ 2
99
+ ],
100
+ "fix_std": 0,
101
+ "layer_scale_init_value": 1e-06,
102
+ "layernorm": "RMSNorm",
103
+ "layernorm_elementwise_affine": true,
104
+ "layernorm_eps": 1e-05,
105
+ "mixer_layer": "depthwise_conv",
106
+ "model_type": "vibevoice_semantic_tokenizer",
107
+ "pad_mode": "constant",
108
+ "std_dist_type": "none",
109
+ "vae_dim": 128,
110
+ "weight_init_value": 0.01
111
+ },
112
+ "semantic_vae_dim": 128,
113
+ "tie_word_embeddings": false,
114
+ "torch_dtype": "bfloat16",
115
+ "transformers_version": "4.51.3",
116
+ "quantization_config": {
117
+ "quant_method": "bitsandbytes",
118
+ "_load_in_8bit": false,
119
+ "_load_in_4bit": true,
120
+ "llm_int8_threshold": 6.0,
121
+ "llm_int8_skip_modules": null,
122
+ "llm_int8_enable_fp32_cpu_offload": false,
123
+ "llm_int8_has_fp16_weight": false,
124
+ "bnb_4bit_quant_type": "nf4",
125
+ "bnb_4bit_use_double_quant": true,
126
+ "bnb_4bit_compute_dtype": "bfloat16",
127
+ "bnb_4bit_quant_storage": "uint8",
128
+ "load_in_4bit": true,
129
+ "load_in_8bit": false
130
+ },
131
+ "_quantization_method": "bitsandbytes"
132
+ }
4bit/generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "transformers_version": "4.51.3"
4
+ }
4bit/load_quantized_4bit.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ """
3
+ Load and use the 4-bit quantized VibeVoice model
4
+ """
5
+
6
+ import torch
7
+ from transformers import BitsAndBytesConfig
8
+ from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
9
+ from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
10
+
11
+ def load_quantized_model(model_path="/home/deveraux/Desktop/vibevoice/VibeVoice-Large-4bit"):
12
+ """Load the pre-quantized VibeVoice model"""
13
+
14
+ print("Loading 4-bit quantized VibeVoice model...")
15
+
16
+ # The model is already quantized, but we need to specify the config
17
+ # to ensure proper loading of quantized weights
18
+ bnb_config = BitsAndBytesConfig(
19
+ load_in_4bit=True,
20
+ bnb_4bit_compute_dtype=torch.bfloat16,
21
+ bnb_4bit_use_double_quant=True,
22
+ bnb_4bit_quant_type='nf4'
23
+ )
24
+
25
+ # Load processor
26
+ processor = VibeVoiceProcessor.from_pretrained(model_path)
27
+
28
+ # Load model
29
+ model = VibeVoiceForConditionalGenerationInference.from_pretrained(
30
+ model_path,
31
+ quantization_config=bnb_config,
32
+ device_map='cuda',
33
+ torch_dtype=torch.bfloat16,
34
+ )
35
+
36
+ model.eval()
37
+
38
+ print("✅ Model loaded successfully!")
39
+ print(f"💾 Memory usage: {torch.cuda.memory_allocated() / 1e9:.1f} GB")
40
+
41
+ return model, processor
42
+
43
+ # Example usage
44
+ if __name__ == "__main__":
45
+ model, processor = load_quantized_model()
46
+
47
+ # Generate audio
48
+ text = "Speaker 1: Hello! Speaker 2: Hi there!"
49
+ inputs = processor(
50
+ text=[text],
51
+ voice_samples=[["path/to/voice1.wav", "path/to/voice2.wav"]],
52
+ padding=True,
53
+ return_tensors="pt",
54
+ )
55
+
56
+ with torch.no_grad():
57
+ outputs = model.generate(**inputs)
58
+
59
+ # Save audio
60
+ processor.save_audio(outputs.speech_outputs[0], "output.wav")
4bit/model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb5b251fc215bcd5ee9d74bc4383aece74ab8a4e38d402017bcf349c20cc02a6
3
+ size 4949701350
4bit/model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40b3e0fe570de93c87842f6bca65a4c4012e41920a6a5342b967206db781b603
3
+ size 1676476837
4bit/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
4bit/preprocessor_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "processor_class": "VibeVoiceProcessor",
3
+ "speech_tok_compress_ratio": 3200,
4
+ "db_normalize": true,
5
+ "audio_processor": {
6
+ "feature_extractor_type": "VibeVoiceTokenizerProcessor",
7
+ "sampling_rate": 24000,
8
+ "normalize_audio": true,
9
+ "target_dB_FS": -25,
10
+ "eps": 1e-06
11
+ }
12
+ }
4bit/quantization_config.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "quantization_config": {
3
+ "quant_method": "bitsandbytes",
4
+ "_load_in_8bit": false,
5
+ "_load_in_4bit": true,
6
+ "llm_int8_threshold": 6.0,
7
+ "llm_int8_skip_modules": null,
8
+ "llm_int8_enable_fp32_cpu_offload": false,
9
+ "llm_int8_has_fp16_weight": false,
10
+ "bnb_4bit_quant_type": "nf4",
11
+ "bnb_4bit_use_double_quant": true,
12
+ "bnb_4bit_compute_dtype": "bfloat16",
13
+ "bnb_4bit_quant_storage": "uint8",
14
+ "load_in_4bit": true,
15
+ "load_in_8bit": false
16
+ },
17
+ "quantization_method": "bitsandbytes",
18
+ "bits": 4,
19
+ "quant_type": "nf4"
20
+ }
4bit/test_output.wav ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:950196552c3c539a6169f5db79fbaf61106945adcdc102e9739c5ec7dc02d83c
3
+ size 236844
8bit/README.md ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # VibeVoice 7B - 8-bit Quantized
2
+
3
+ Better quality with moderate VRAM requirements.
4
+
5
+ ## Specifications
6
+ - Quantization: 8-bit (int8)
7
+ - Model size: 9.9 GB
8
+ - VRAM usage: ~12 GB
9
+ - Quality: Excellent (minimal degradation)
10
+
11
+ ## Usage
12
+
13
+ ```python
14
+ from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
15
+ from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
16
+
17
+ model = VibeVoiceForConditionalGenerationInference.from_pretrained(
18
+ "Dannidee/VibeVoice7b-low-vram/8bit",
19
+ device_map='cuda',
20
+ torch_dtype=torch.bfloat16,
21
+ )
22
+ processor = VibeVoiceProcessor.from_pretrained("Dannidee/VibeVoice7b-low-vram/8bit")
23
+ ```
8bit/config.json ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "acoustic_vae_dim": 64,
3
+ "acoustic_tokenizer_config": {
4
+ "causal": true,
5
+ "channels": 1,
6
+ "conv_bias": true,
7
+ "conv_norm": "none",
8
+ "corpus_normalize": 0.0,
9
+ "decoder_depths": null,
10
+ "decoder_n_filters": 32,
11
+ "decoder_ratios": [
12
+ 8,
13
+ 5,
14
+ 5,
15
+ 4,
16
+ 2,
17
+ 2
18
+ ],
19
+ "disable_last_norm": true,
20
+ "encoder_depths": "3-3-3-3-3-3-8",
21
+ "encoder_n_filters": 32,
22
+ "encoder_ratios": [
23
+ 8,
24
+ 5,
25
+ 5,
26
+ 4,
27
+ 2,
28
+ 2
29
+ ],
30
+ "fix_std": 0.5,
31
+ "layer_scale_init_value": 1e-06,
32
+ "layernorm": "RMSNorm",
33
+ "layernorm_elementwise_affine": true,
34
+ "layernorm_eps": 1e-05,
35
+ "mixer_layer": "depthwise_conv",
36
+ "model_type": "vibevoice_acoustic_tokenizer",
37
+ "pad_mode": "constant",
38
+ "std_dist_type": "gaussian",
39
+ "vae_dim": 64,
40
+ "weight_init_value": 0.01
41
+ },
42
+ "architectures": [
43
+ "VibeVoiceForConditionalGeneration"
44
+ ],
45
+ "decoder_config": {
46
+ "attention_dropout": 0.0,
47
+ "hidden_act": "silu",
48
+ "hidden_size": 3584,
49
+ "initializer_range": 0.02,
50
+ "intermediate_size": 18944,
51
+ "max_position_embeddings": 32768,
52
+ "max_window_layers": 28,
53
+ "model_type": "qwen2",
54
+ "num_attention_heads": 28,
55
+ "num_hidden_layers": 28,
56
+ "num_key_value_heads": 4,
57
+ "rms_norm_eps": 1e-06,
58
+ "rope_scaling": null,
59
+ "rope_theta": 1000000.0,
60
+ "sliding_window": null,
61
+ "torch_dtype": "bfloat16",
62
+ "use_cache": true,
63
+ "use_mrope": false,
64
+ "use_sliding_window": false,
65
+ "vocab_size": 152064
66
+ },
67
+ "diffusion_head_config": {
68
+ "ddpm_batch_mul": 4,
69
+ "ddpm_beta_schedule": "cosine",
70
+ "ddpm_num_inference_steps": 20,
71
+ "ddpm_num_steps": 1000,
72
+ "diffusion_type": "ddpm",
73
+ "head_ffn_ratio": 3.0,
74
+ "head_layers": 4,
75
+ "hidden_size": 3584,
76
+ "latent_size": 64,
77
+ "model_type": "vibevoice_diffusion_head",
78
+ "prediction_type": "v_prediction",
79
+ "rms_norm_eps": 1e-05,
80
+ "speech_vae_dim": 64
81
+ },
82
+ "model_type": "vibevoice",
83
+ "semantic_tokenizer_config": {
84
+ "causal": true,
85
+ "channels": 1,
86
+ "conv_bias": true,
87
+ "conv_norm": "none",
88
+ "corpus_normalize": 0.0,
89
+ "disable_last_norm": true,
90
+ "encoder_depths": "3-3-3-3-3-3-8",
91
+ "encoder_n_filters": 32,
92
+ "encoder_ratios": [
93
+ 8,
94
+ 5,
95
+ 5,
96
+ 4,
97
+ 2,
98
+ 2
99
+ ],
100
+ "fix_std": 0,
101
+ "layer_scale_init_value": 1e-06,
102
+ "layernorm": "RMSNorm",
103
+ "layernorm_elementwise_affine": true,
104
+ "layernorm_eps": 1e-05,
105
+ "mixer_layer": "depthwise_conv",
106
+ "model_type": "vibevoice_semantic_tokenizer",
107
+ "pad_mode": "constant",
108
+ "std_dist_type": "none",
109
+ "vae_dim": 128,
110
+ "weight_init_value": 0.01
111
+ },
112
+ "semantic_vae_dim": 128,
113
+ "tie_word_embeddings": false,
114
+ "torch_dtype": "bfloat16",
115
+ "transformers_version": "4.51.3",
116
+ "quantization_config": {
117
+ "quant_method": "bitsandbytes",
118
+ "_load_in_8bit": true,
119
+ "_load_in_4bit": false,
120
+ "llm_int8_threshold": 6.0,
121
+ "llm_int8_skip_modules": null,
122
+ "llm_int8_enable_fp32_cpu_offload": false,
123
+ "llm_int8_has_fp16_weight": false,
124
+ "bnb_4bit_quant_type": "fp4",
125
+ "bnb_4bit_use_double_quant": false,
126
+ "bnb_4bit_compute_dtype": "float32",
127
+ "bnb_4bit_quant_storage": "uint8",
128
+ "load_in_4bit": false,
129
+ "load_in_8bit": true
130
+ },
131
+ "_quantization_method": "bitsandbytes"
132
+ }
8bit/generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "transformers_version": "4.51.3"
4
+ }
8bit/load_quantized_8bit.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+ """
3
+ Load and use the 8-bit quantized VibeVoice model
4
+ """
5
+
6
+ import torch
7
+ from transformers import BitsAndBytesConfig
8
+ from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
9
+ from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
10
+
11
+ def load_quantized_model(model_path="/home/deveraux/Desktop/vibevoice/VibeVoice-Large-8bit"):
12
+ """Load the pre-quantized VibeVoice model"""
13
+
14
+ print("Loading 8-bit quantized VibeVoice model...")
15
+
16
+ # The model is already quantized, but we need to specify the config
17
+ # to ensure proper loading of quantized weights
18
+ bnb_config = BitsAndBytesConfig(
19
+ load_in_8bit=True,
20
+ bnb_8bit_compute_dtype=torch.bfloat16,
21
+
22
+
23
+ )
24
+
25
+ # Load processor
26
+ processor = VibeVoiceProcessor.from_pretrained(model_path)
27
+
28
+ # Load model
29
+ model = VibeVoiceForConditionalGenerationInference.from_pretrained(
30
+ model_path,
31
+ quantization_config=bnb_config,
32
+ device_map='cuda',
33
+ torch_dtype=torch.bfloat16,
34
+ )
35
+
36
+ model.eval()
37
+
38
+ print("✅ Model loaded successfully!")
39
+ print(f"💾 Memory usage: {torch.cuda.memory_allocated() / 1e9:.1f} GB")
40
+
41
+ return model, processor
42
+
43
+ # Example usage
44
+ if __name__ == "__main__":
45
+ model, processor = load_quantized_model()
46
+
47
+ # Generate audio
48
+ text = "Speaker 1: Hello! Speaker 2: Hi there!"
49
+ inputs = processor(
50
+ text=[text],
51
+ voice_samples=[["path/to/voice1.wav", "path/to/voice2.wav"]],
52
+ padding=True,
53
+ return_tensors="pt",
54
+ )
55
+
56
+ with torch.no_grad():
57
+ outputs = model.generate(**inputs)
58
+
59
+ # Save audio
60
+ processor.save_audio(outputs.speech_outputs[0], "output.wav")
8bit/model-00001-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68f98075dac463766219e6e61ff5fe9ab969f8fea621a65906f1d6793f2eaf72
3
+ size 4987685394
8bit/model-00002-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48940fb59366de226af5df46020f022d4d651f4563f190142c175b5bf733e9c7
3
+ size 4489976774
8bit/model-00003-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d83c0514c0c9d2675cb4d51ee56b12515ea45770ce35acc5ab0ec4bc7d1bef73
3
+ size 1089994880
8bit/model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
8bit/preprocessor_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "processor_class": "VibeVoiceProcessor",
3
+ "speech_tok_compress_ratio": 3200,
4
+ "db_normalize": true,
5
+ "audio_processor": {
6
+ "feature_extractor_type": "VibeVoiceTokenizerProcessor",
7
+ "sampling_rate": 24000,
8
+ "normalize_audio": true,
9
+ "target_dB_FS": -25,
10
+ "eps": 1e-06
11
+ }
12
+ }
8bit/quantization_config.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "quantization_config": {
3
+ "quant_method": "bitsandbytes",
4
+ "_load_in_8bit": true,
5
+ "_load_in_4bit": false,
6
+ "llm_int8_threshold": 6.0,
7
+ "llm_int8_skip_modules": null,
8
+ "llm_int8_enable_fp32_cpu_offload": false,
9
+ "llm_int8_has_fp16_weight": false,
10
+ "bnb_4bit_quant_type": "fp4",
11
+ "bnb_4bit_use_double_quant": false,
12
+ "bnb_4bit_compute_dtype": "float32",
13
+ "bnb_4bit_quant_storage": "uint8",
14
+ "load_in_4bit": false,
15
+ "load_in_8bit": true
16
+ },
17
+ "quantization_method": "bitsandbytes",
18
+ "bits": 8,
19
+ "quant_type": "nf4"
20
+ }
README.md CHANGED
@@ -1,97 +1,43 @@
1
- Here's some quantized VibeVoice 7b models, both 8 and 4 bit, along with some simple python code to test them out.
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
- ## Model Sizes
4
 
5
- | Model Version | Size | Memory Usage | Quality |
6
- |---------------|------|--------------|---------|
7
- | Original (fp16/bf16) | 18GB | ~18GB VRAM | Best |
8
- | 8-bit Quantized | 9.9GB | ~10.6GB VRAM | Excellent |
9
- | 4-bit Quantized (nf4) | 6.2GB | ~6.6GB VRAM | Very Good |
10
 
11
- ## How to Use Pre-Quantized Models
 
12
 
13
- ### 1. Loading 4-bit Model
14
 
15
  ```python
16
  from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
17
  from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
18
 
19
- # Load pre-quantized 4-bit model
20
- model_path = "/path/to/VibeVoice-Large-4bit"
21
- processor = VibeVoiceProcessor.from_pretrained(model_path)
22
  model = VibeVoiceForConditionalGenerationInference.from_pretrained(
23
- model_path,
24
  device_map='cuda',
25
  torch_dtype=torch.bfloat16,
26
  )
 
27
  ```
28
 
29
- ### 2. Loading 8-bit Model
30
-
31
- ```python
32
- # Same code, just point to 8-bit model
33
- model_path = "/path/to/VibeVoice-Large-8bit"
34
- # ... rest is the same
35
- ```
36
-
37
- ## Creating Your Own Quantized Models
38
-
39
- Use the provided script to quantize models:
40
-
41
- ```bash
42
- # 4-bit quantization (nf4)
43
- python quantize_and_save_vibevoice.py \
44
- --model_path /path/to/original/model \
45
- --output_dir /path/to/output/4bit \
46
- --bits 4 \
47
- --test
48
-
49
- # 8-bit quantization
50
- python quantize_and_save_vibevoice.py \
51
- --model_path /path/to/original/model \
52
- --output_dir /path/to/output/8bit \
53
- --bits 8 \
54
- --test
55
- ```
56
-
57
- ## Benefits
58
 
59
- 1. **Pre-quantized models load faster** - No on-the-fly quantization needed
60
- 2. **Lower VRAM requirements** - 4-bit uses only ~6.6GB vs 18GB
61
- 3. **Shareable** - Upload the quantized folder to share with others
62
- 4. **Quality preserved** - nf4 quantization maintains excellent output quality
63
 
64
- ## Distribution
65
-
66
- To share quantized models:
67
-
68
- 1. Upload the entire quantized model directory (e.g., `VibeVoice-Large-4bit/`)
69
- 2. Include the `quantization_config.json` file (automatically created)
70
- 3. Users can load directly without any quantization setup
71
-
72
- ## Performance Notes
73
-
74
- - 4-bit (nf4): Best for memory-constrained systems, minimal quality loss
75
- - 8-bit: Better quality than 4-bit, still significant memory savings
76
- - Both versions maintain the same generation speed as the original
77
- - Flash Attention 2 is supported in all quantized versions
78
-
79
- ## Troubleshooting
80
-
81
- If loading fails:
82
- 1. Ensure you have `bitsandbytes` installed: `pip install bitsandbytes`
83
- 2. Make sure you're on a CUDA-capable GPU
84
- 3. Check that all model files are present in the directory
85
-
86
- ## Files Created
87
-
88
- Each quantized model directory contains:
89
- - `model.safetensors.*` - Quantized model weights
90
- - `config.json` - Model configuration with quantization settings
91
- - `quantization_config.json` - Specific quantization parameters
92
- - `processor/` - Audio processor files
93
- - `load_quantized_Xbit.py` - Example loading script
94
-
95
- ---
96
- license: mit
97
- ---
 
1
+ ---
2
+ license: other
3
+ language:
4
+ - en
5
+ tags:
6
+ - text-to-speech
7
+ - speech-synthesis
8
+ - quantized
9
+ - low-vram
10
+ - vibevoice
11
+ ---
12
+
13
+ # VibeVoice 7B - Low VRAM Quantized Models
14
 
15
+ Pre-quantized versions of VibeVoice 7B for low VRAM GPUs.
16
 
17
+ ## Available Versions
 
 
 
 
18
 
19
+ - **4bit/** - 4-bit quantized model (~8GB VRAM needed)
20
+ - **8bit/** - 8-bit quantized model (~12GB VRAM needed)
21
 
22
+ ## Usage
23
 
24
  ```python
25
  from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
26
  from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
27
 
28
+ # For 4-bit model
 
 
29
  model = VibeVoiceForConditionalGenerationInference.from_pretrained(
30
+ "Dannidee/VibeVoice7b-low-vram/4bit",
31
  device_map='cuda',
32
  torch_dtype=torch.bfloat16,
33
  )
34
+ processor = VibeVoiceProcessor.from_pretrained("Dannidee/VibeVoice7b-low-vram/4bit")
35
  ```
36
 
37
+ ## VRAM Requirements
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
 
39
+ - **4-bit**: ~8 GB total VRAM
40
+ - **8-bit**: ~12 GB total VRAM
41
+ - **Original**: ~19 GB total VRAM
 
42
 
43
+ See individual model folders for detailed information.