Upload tokenizer
Browse files- added_tokens.json +12 -12
added_tokens.json
CHANGED
|
@@ -37,16 +37,16 @@
|
|
| 37 |
" ": 50259,
|
| 38 |
" ": 50258,
|
| 39 |
" ": 50257,
|
| 40 |
-
"<PAD>":
|
| 41 |
-
"<|end_of_turn|>":
|
| 42 |
-
"<|end|>":
|
| 43 |
-
"<|function_call|>":
|
| 44 |
-
"<|response|>":
|
| 45 |
-
"<|thoughts|>":
|
| 46 |
-
"Azma":
|
| 47 |
-
"Azma:":
|
| 48 |
-
"Observation:":
|
| 49 |
-
"Reference:":
|
| 50 |
-
"System:":
|
| 51 |
-
"User:":
|
| 52 |
}
|
|
|
|
| 37 |
" ": 50259,
|
| 38 |
" ": 50258,
|
| 39 |
" ": 50257,
|
| 40 |
+
"<PAD>": 50296,
|
| 41 |
+
"<|end_of_turn|>": 50298,
|
| 42 |
+
"<|end|>": 50297,
|
| 43 |
+
"<|function_call|>": 50301,
|
| 44 |
+
"<|response|>": 50305,
|
| 45 |
+
"<|thoughts|>": 50304,
|
| 46 |
+
"Azma": 50303,
|
| 47 |
+
"Azma:": 50302,
|
| 48 |
+
"Observation:": 50299,
|
| 49 |
+
"Reference:": 50295,
|
| 50 |
+
"System:": 50300,
|
| 51 |
+
"User:": 50306
|
| 52 |
}
|