fix can't set attribute 'eos_token' when loading the saved tokenizer (#27)

- fix can't set attribute 'eos_token' when loading the saved tokenizer (72e7f646bc14c58534be3abd4001116bf20c18cc)


Co-authored-by: hoshi hiyouga <[email protected]>

tokenization_chatglm.py

@@ -8,6 +8,9 @@ from transformers.utils import logging, PaddingStrategy
 from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
 
 
+logger = logging.get_logger(__name__)
+
+
 class SPTokenizer:
     def __init__(self, model_path: str):
         # reload tokenizer
@@ -89,25 +92,34 @@ class SPTokenizer:
 
 
 class ChatGLMTokenizer(PreTrainedTokenizer):
-    vocab_files_names = {"vocab_file": "tokenizer.model"}
 
+    vocab_files_names = {"vocab_file": "tokenizer.model"}
     model_input_names = ["input_ids", "attention_mask", "position_ids"]
 
-    def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, encode_special_tokens=False,
-                 **kwargs):
+    def __init__(
+        self,
+        vocab_file,
+        padding_side="left",
+        clean_up_tokenization_spaces=False,
+        encode_special_tokens=False,
+        **kwargs
+    ):
         self.name = "GLMTokenizer"
-
         self.vocab_file = vocab_file
         self.tokenizer = SPTokenizer(vocab_file)
         self.special_tokens = {
             "<bos>": self.tokenizer.bos_id,
             "<eos>": self.tokenizer.eos_id,
+            "<unk>": self.tokenizer.pad_id,
             "<pad>": self.tokenizer.pad_id
         }
         self.encode_special_tokens = encode_special_tokens
-        super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-                         encode_special_tokens=encode_special_tokens,
-                         **kwargs)
+
+        super().__init__(
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs
+        )
 
     def get_command(self, token):
         if token in self.special_tokens:
@@ -117,24 +129,40 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
 
     @property
     def unk_token(self) -> str:
-        return "<unk>"
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<unk>"))
 
     @property
     def pad_token(self) -> str:
-        return "<unk>"
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<pad>"))
 
     @property
-    def pad_token_id(self):
-        return self.get_command("<pad>")
+    def eos_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<eos>"))
 
     @property
-    def eos_token(self) -> str:
-        return "</s>"
+    def unk_token_id(self) -> int:
+        return self.get_command("<unk>")
+
+    @property
+    def pad_token_id(self) -> int:
+        return self.get_command("<pad>")
 
     @property
     def eos_token_id(self):
         return self.get_command("<eos>")
 
+    @unk_token.setter
+    def unk_token(self, value):
+        logger.warning("Setting unk_token is not supported, use the default one.")
+
+    @pad_token.setter
+    def pad_token(self, value):
+        logger.warning("Setting pad_token is not supported, use the default one.")
+
+    @eos_token.setter
+    def eos_token(self, value):
+        logger.warning("Setting eos_token is not supported, use the default one.")
+
     @property
     def vocab_size(self):
         return self.tokenizer.n_words
@@ -212,7 +240,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
         return self.batch_encode_plus([input_ids], return_tensors="pt", is_split_into_words=True)
 
     def build_inputs_with_special_tokens(
-            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:
         """
         Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
@@ -237,12 +265,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
         return token_ids_0
 
     def _pad(
-            self,
-            encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
-            max_length: Optional[int] = None,
-            padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-            pad_to_multiple_of: Optional[int] = None,
-            return_attention_mask: Optional[bool] = None,
+        self,
+        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+        max_length: Optional[int] = None,
+        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+        pad_to_multiple_of: Optional[int] = None,
+        return_attention_mask: Optional[bool] = None,
     ) -> dict:
         """
         Pad encoded inputs (on left/right and up to predefined length or max length in the batch)