yo

custom_tokenizer.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from transformers import PreTrainedTokenizer, AddedToken
 
 class CustomTokenizer(PreTrainedTokenizer):
@@ -7,4 +8,7 @@ class CustomTokenizer(PreTrainedTokenizer):
 
     def tokenize(self, text):
         print("Tokenizing text", text)
-        return text.split()
+        return text.split()
+
+    def get_vocab(self) -> Dict[str, int]:
+        return {}