Alexis-Az
/

Story-Generation-Model

Text Generation

text-generation-inference

Model card Files Files and versions

Alexis-Az commited on Feb 24

Commit

605d89f

·

verified ·

1 Parent(s): 9b912e6

Upload tokenizer

Files changed (1) hide show

tokenizer_config.json +4 -2

tokenizer_config.json CHANGED Viewed

@@ -1,4 +1,5 @@
 {
   "added_tokens_decoder": {
     "128000": {
       "content": "<|begin_of_text|>",
@@ -2060,6 +2061,7 @@
   ],
   "model_max_length": 131072,
   "pad_token": "<|finetune_right_pad_id|>",
-  "padding_side": "right",
-  "tokenizer_class": "PreTrainedTokenizerFast"
 }

 {
+  "add_bos_token": true,
   "added_tokens_decoder": {
     "128000": {
       "content": "<|begin_of_text|>",
   ],
   "model_max_length": 131072,
   "pad_token": "<|finetune_right_pad_id|>",
+  "padding_side": "left",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": null
 }