add tokenizer

added_tokens.json

@@ -1 +1,3 @@
-{"<hl>": 50265}
+{
+  "<hl>": 50265
+}

special_tokens_map.json

@@ -1 +1,18 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["<hl>"]}
+{
+  "additional_special_tokens": [
+    "<hl>"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -62,7 +62,8 @@
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "post_processor": {
     "type": "RobertaProcessing",
@@ -80,7 +81,8 @@
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "model": {
     "type": "BPE",

tokenizer_config.json

@@ -1 +1,16 @@
-{"errors": "replace", "bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "cls_token": "<s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>", "add_prefix_space": false, "trim_offsets": true, "model_max_length": 1024, "name_or_path": "lmqg_output/bart_base_tweetqa/model_zwjvpy/epoch_10", "special_tokens_map_file": "lmqg_output/bart_base_tweetqa/model_zwjvpy/epoch_10/special_tokens_map.json", "tokenizer_class": "BartTokenizer"}
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 1024,
+  "name_or_path": "bart-base-tweetqa-qag",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": "lmqg_output/bart_base_tweetqa/model_zwjvpy/epoch_10/special_tokens_map.json",
+  "tokenizer_class": "BartTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}