Hugging Face's logo

hongyuw
/
bitnet-b1.58-2B-4T-bf16

Text Generation
Transformers
PyTorch
Safetensors
English
BitNet
chat
bitnet
large-language-model
conversational
custom_code
Model card Files
xet
Community
hongyuw commited on
Commit
733aa2d
·
verified ·
1 Parent(s): abef245

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. config.json +26 -29
  2. generation_config.json +3 -3
  3. pytorch_model.bin +3 -0
  4. torchscale-checkpoint.pt +3 -0
config.json CHANGED
@@ -1,31 +1,28 @@
1
  {
2
- "architectures": [
3
- "BitNetForCausalLM"
4
- ],
5
- "auto_map": {
6
- "AutoConfig": "configuration_bitnet.BitNetConfig",
7
- "AutoModelForCausalLM": "modeling_bitnet.BitNetForCausalLM"
8
- },
9
- "bos_token_id": 128000,
10
- "eos_token_id": 128001,
11
- "hidden_act": "relu2",
12
- "hidden_size": 2560,
13
- "initializer_range": 0.02,
14
- "intermediate_size": 6912,
15
- "max_position_embeddings": 4096,
16
- "model_type": "bitnet",
17
- "rms_norm_eps": 1e-05,
18
- "num_attention_heads": 20,
19
- "num_hidden_layers": 30,
20
- "num_key_value_heads": 5,
21
- "rope_theta": 500000.0,
22
- "tie_word_embeddings": true,
23
- "torch_dtype": "bfloat16",
24
- "use_cache": true,
25
- "vocab_size": 128256,
26
- "quantization_config": {
27
- "quant_method": "bitnet",
28
- "linear_class": "autobitlinear",
29
- "quantization_mode": "online"
30
- }
31
  }
 
1
  {
2
+ "architectures": [
3
+ "BitNetForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_bitnet.BitNetConfig",
9
+ "AutoModelForCausalLM": "modeling_bitnet.BitNetForCausalLM"
10
+ },
11
+ "bos_token_id": 128000,
12
+ "eos_token_id": 128001,
13
+ "hidden_act": "relu2",
14
+ "hidden_size": 2560,
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 6912,
17
+ "max_position_embeddings": 4096,
18
+ "model_type": "BitNet",
19
+ "num_attention_heads": 20,
20
+ "num_hidden_layers": 30,
21
+ "num_key_value_heads": 5,
22
+ "rms_norm_eps": 1e-05,
23
+ "rope_theta": 500000.0,
24
+ "torch_dtype": "float32",
25
+ "transformers_version": "4.51.0.dev0",
26
+ "use_cache": true,
27
+ "vocab_size": 128256
 
 
 
28
  }
generation_config.json CHANGED
@@ -1,12 +1,12 @@
1
  {
2
  "bos_token_id": 128000,
 
3
  "eos_token_id": [
4
  128001,
5
  128009
6
  ],
7
- "do_sample": true,
8
- "temperature": 0.6,
9
  "max_length": 4096,
 
10
  "top_p": 0.9,
11
- "transformers_version": "4.40.0.dev0"
12
  }
 
1
  {
2
  "bos_token_id": 128000,
3
+ "do_sample": true,
4
  "eos_token_id": [
5
  128001,
6
  128009
7
  ],
 
 
8
  "max_length": 4096,
9
+ "temperature": 0.6,
10
  "top_p": 0.9,
11
+ "transformers_version": "4.51.0.dev0"
12
  }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3063f9abb34250e25ecbc97ed104731ab733a078663c693b329dc6ce2c4a750a
3
+ size 9651392138
torchscale-checkpoint.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41b67a2d6c5aabb741e96b7fffb5011bb300c9f2081b43ce5309b5e4b499b98a
3
+ size 9651395390