Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
|
@@ -991,7 +991,7 @@
|
|
| 991 |
{
|
| 992 |
"type": "Metaspace",
|
| 993 |
"replacement": "▁",
|
| 994 |
-
"
|
| 995 |
},
|
| 996 |
{
|
| 997 |
"type": "Split",
|
|
@@ -1104,7 +1104,7 @@
|
|
| 1104 |
{
|
| 1105 |
"type": "Metaspace",
|
| 1106 |
"replacement": "▁",
|
| 1107 |
-
"
|
| 1108 |
},
|
| 1109 |
{
|
| 1110 |
"type": "Strip",
|
|
|
|
| 991 |
{
|
| 992 |
"type": "Metaspace",
|
| 993 |
"replacement": "▁",
|
| 994 |
+
"prepend_scheme": "always"
|
| 995 |
},
|
| 996 |
{
|
| 997 |
"type": "Split",
|
|
|
|
| 1104 |
{
|
| 1105 |
"type": "Metaspace",
|
| 1106 |
"replacement": "▁",
|
| 1107 |
+
"prepend_scheme": "never"
|
| 1108 |
},
|
| 1109 |
{
|
| 1110 |
"type": "Strip",
|