    def intra_word_tokenize(
        self, string_tokens: List[str]
    ) -> Tuple[List[Token], List[Optional[Tuple[int, int]]]]:
        """
        Tokenizes each word into wordpieces separately and returns the wordpiece IDs.
        Also calculates offsets such that tokens[offsets[i][0]:offsets[i][1] + 1]
        corresponds to the original i-th token.

        This function inserts special tokens.
        """
        tokens, offsets = self._intra_word_tokenize(string_tokens)
        tokens = self.add_special_tokens(tokens)
        offsets = self._increment_offsets(offsets, len(self.single_sequence_start_tokens))
        return tokens, offsets
Tokenizes each word into wordpieces separately and returns the wordpiece IDs.
Also calculates offsets such that tokens[offsets[i][0]:offsets[i][1] + 1]
corresponds to the original i-th token.

This function inserts special tokens.
allennlp/data/tokenizers/pretrained_transformer_tokenizer.py