Gustavo de Rosa
commited on
Commit
·
492fc70
1
Parent(s):
b15fb48
chore(root): Changes pad_token to <|dummy_85|>.
Browse files- config.json +1 -1
- generation_config.json +1 -1
- special_tokens_map.json +1 -1
- tokenizer_config.json +1 -1
config.json
CHANGED
@@ -19,7 +19,7 @@
|
|
19 |
"num_hidden_layers": 40,
|
20 |
"num_key_value_heads": 10,
|
21 |
"original_max_position_embeddings": 16384,
|
22 |
-
"pad_token_id":
|
23 |
"resid_pdrop": 0.0,
|
24 |
"rms_norm_eps": 1e-05,
|
25 |
"rope_scaling": null,
|
|
|
19 |
"num_hidden_layers": 40,
|
20 |
"num_key_value_heads": 10,
|
21 |
"original_max_position_embeddings": 16384,
|
22 |
+
"pad_token_id": 100349,
|
23 |
"resid_pdrop": 0.0,
|
24 |
"rms_norm_eps": 1e-05,
|
25 |
"rope_scaling": null,
|
generation_config.json
CHANGED
@@ -2,6 +2,6 @@
|
|
2 |
"_from_model_config": true,
|
3 |
"bos_token_id": 100257,
|
4 |
"eos_token_id": 100265,
|
5 |
-
"pad_token_id":
|
6 |
"transformers_version": "4.47.0"
|
7 |
}
|
|
|
2 |
"_from_model_config": true,
|
3 |
"bos_token_id": 100257,
|
4 |
"eos_token_id": 100265,
|
5 |
+
"pad_token_id": 100349,
|
6 |
"transformers_version": "4.47.0"
|
7 |
}
|
special_tokens_map.json
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
{
|
2 |
"bos_token": "<|endoftext|>",
|
3 |
"eos_token": "<|im_end|>",
|
4 |
-
"pad_token": "<|
|
5 |
}
|
|
|
1 |
{
|
2 |
"bos_token": "<|endoftext|>",
|
3 |
"eos_token": "<|im_end|>",
|
4 |
+
"pad_token": "<|dummy_85|>"
|
5 |
}
|
tokenizer_config.json
CHANGED
@@ -775,6 +775,6 @@
|
|
775 |
"clean_up_tokenization_spaces": false,
|
776 |
"eos_token": "<|im_end|>",
|
777 |
"model_max_length": 16384,
|
778 |
-
"pad_token": "<|
|
779 |
"tokenizer_class": "GPT2Tokenizer"
|
780 |
}
|
|
|
775 |
"clean_up_tokenization_spaces": false,
|
776 |
"eos_token": "<|im_end|>",
|
777 |
"model_max_length": 16384,
|
778 |
+
"pad_token": "<|dummy_85|>",
|
779 |
"tokenizer_class": "GPT2Tokenizer"
|
780 |
}
|