Asaf Yehoodai
commited on
Commit
·
0a28e2f
1
Parent(s):
6bccc43
add tokenizer
Browse files- added_tokens.json +1 -0
- source.spm +0 -0
- special_tokens_map.json +1 -0
- target.spm +0 -0
- tokenizer_config.json +1 -0
- vocab.json +0 -0
added_tokens.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<ENT0>": 59514, "</ENT0>": 59515, "<ENT1>": 59516, "</ENT1>": 59517, "<ENT2>": 59518, "</ENT2>": 59519, "<ENT3>": 59520, "</ENT3>": 59521, "<ENT4>": 59522, "</ENT4>": 59523, "<ENT5>": 59524, "</ENT5>": 59525, "<ENT6>": 59526, "</ENT6>": 59527, "<ENT7>": 59528, "</ENT7>": 59529, "<ENT8>": 59530, "</ENT8>": 59531, "<ENT9>": 59532, "</ENT9>": 59533, "<ENT10>": 59534, "</ENT10>": 59535, "<ENT11>": 59536, "</ENT11>": 59537, "<ENT12>": 59538, "</ENT12>": 59539, "<ENT13>": 59540, "</ENT13>": 59541, "<ENT14>": 59542, "</ENT14>": 59543, "<ENT15>": 59544, "</ENT15>": 59545, "<ENT16>": 59546, "</ENT16>": 59547, "<ENT17>": 59548, "</ENT17>": 59549, "<ENT18>": 59550, "</ENT18>": 59551, "<ENT19>": 59552, "</ENT19>": 59553, "<MALE>": 59554, "</MALE>": 59555, "<FEMALE>": 59556, "</FEMALE>": 59557}
|
source.spm
ADDED
|
Binary file (778 kB). View file
|
|
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
|
target.spm
ADDED
|
Binary file (802 kB). View file
|
|
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"source_lang": "en", "target_lang": "fr", "unk_token": "<unk>", "eos_token": "</s>", "pad_token": "<pad>", "model_max_length": 512, "sp_model_kwargs": {}, "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "/cs/labs/oabend/asaf305612939/co-ref-project/mt-output/paper_models/fr/en_fr_gender", "tokenizer_class": "MarianTokenizer"}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|