zequnl commited on
Commit
fa3985d
·
verified ·
1 Parent(s): 7e91828

Update molxpt_tokenizer.py

Browse files
Files changed (1) hide show
  1. molxpt_tokenizer.py +4 -2
molxpt_tokenizer.py CHANGED
@@ -1,7 +1,7 @@
1
  from transformers import BioGptTokenizer, AutoTokenizer
2
  import re
3
 
4
- AutoTokenizer.register(MolxptTokenizer)
5
 
6
  class MolxptTokenizer(BioGptTokenizer):
7
  def __init__(
@@ -73,4 +73,6 @@ class MolxptTokenizer(BioGptTokenizer):
73
  else:
74
  new_splits.append(s)
75
  text = " ".join(new_splits)
76
- return text
 
 
 
1
  from transformers import BioGptTokenizer, AutoTokenizer
2
  import re
3
 
4
+
5
 
6
  class MolxptTokenizer(BioGptTokenizer):
7
  def __init__(
 
73
  else:
74
  new_splits.append(s)
75
  text = " ".join(new_splits)
76
+ return text
77
+
78
+ AutoTokenizer.register(MolxptTokenizer)