hhou435 commited on
Commit
fafda48
·
1 Parent(s): dafbf79
Files changed (2) hide show
  1. tokenizer_config.json +1 -1
  2. vocab.txt +3 -3
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"do_lower_case": false, "do_basic_tokenize": true, "never_split": ["<extra_id_0>","<extra_id_1>","<extra_id_2>"], "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null}
 
1
+ {"do_lower_case": false, "do_basic_tokenize": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null}
vocab.txt CHANGED
@@ -1,7 +1,7 @@
1
  [PAD]
2
- <extra_id_0>
3
- <extra_id_1>
4
- <extra_id_2>
5
  [extra_id_3]
6
  [extra_id_4]
7
  [extra_id_5]
 
1
  [PAD]
2
+ e0
3
+ e1
4
+ e2
5
  [extra_id_3]
6
  [extra_id_4]
7
  [extra_id_5]