Update molxpt_tokenizer.py
Browse files- molxpt_tokenizer.py +4 -2
molxpt_tokenizer.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
from transformers import BioGptTokenizer, AutoTokenizer
|
2 |
import re
|
3 |
|
4 |
-
|
5 |
|
6 |
class MolxptTokenizer(BioGptTokenizer):
|
7 |
def __init__(
|
@@ -73,4 +73,6 @@ class MolxptTokenizer(BioGptTokenizer):
|
|
73 |
else:
|
74 |
new_splits.append(s)
|
75 |
text = " ".join(new_splits)
|
76 |
-
return text
|
|
|
|
|
|
1 |
from transformers import BioGptTokenizer, AutoTokenizer
|
2 |
import re
|
3 |
|
4 |
+
|
5 |
|
6 |
class MolxptTokenizer(BioGptTokenizer):
|
7 |
def __init__(
|
|
|
73 |
else:
|
74 |
new_splits.append(s)
|
75 |
text = " ".join(new_splits)
|
76 |
+
return text
|
77 |
+
|
78 |
+
AutoTokenizer.register(MolxptTokenizer)
|