RichardErkhov commited on
Commit
e43528a
·
verified ·
1 Parent(s): 077e4b1

uploaded model

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +31 -0
tokenizer_config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "50257": {
14
+ "content": "<|stop|>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ }
21
+ },
22
+ "bos_token": "<|endoftext|>",
23
+ "chat_template": "{% set message_count = messages|length %}{% if message_count >= 4 and messages[-4].role == 'user' %}{% set recent_messages = messages[-4:] %}{% elif message_count > 3 %}{% set recent_messages = messages[-3:] %}{% else %}{% set recent_messages = messages %}{% endif %}{% for message in recent_messages %}{% if message.role == 'user' %}### Kullanıcı:\n{{ message.content }}\n{% elif message.role == 'assistant' %}### Asistan:\n{{ message.content }}\n{% endif %}{% endfor %}{% if messages[-1]['role'] == 'user' %}### Asistan:\n{% endif %}",
24
+ "clean_up_tokenization_spaces": true,
25
+ "eos_token": "<|stop|>",
26
+ "errors": "replace",
27
+ "model_max_length": 1000000000000000019884624838656,
28
+ "pad_token": "<|endoftext|>",
29
+ "tokenizer_class": "GPT2Tokenizer",
30
+ "unk_token": "<|endoftext|>"
31
+ }