    def tokenize(self, text):
        """
        Tokenizes a piece of text into its word pieces.
        This uses a greedy longest-match-first algorithm to perform tokenization
        using the given vocabulary.

        Args:
            text: A single token or whitespace separated tokens. This should have
                already been passed through `BasicTokenizer`.

        Returns:
            list (str): A list of wordpiece tokens.

        Examples:
            .. code-block::

                from paddlenlp.transformers import BertTokenizer, WordpieceTokenizer

                berttokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
                vocab  = berttokenizer.vocab
                unk_token = berttokenizer.unk_token

                wordpiecetokenizer = WordpieceTokenizer(vocab,unk_token)
                inputs = wordpiecetokenizer.tokenize("unaffable")
                print(inputs)
                '''
                ["un", "##aff", "##able"]
                '''

        """

        output_tokens = []
        for token in whitespace_tokenize(text):
            chars = list(token)
            if len(chars) > self.max_input_chars_per_word:
                output_tokens.append(self.unk_token)
                continue

            is_bad = False
            start = 0
            sub_tokens = []
            while start < len(chars):
                end = len(chars)
                cur_substr = None
                while start < end:
                    substr = "".join(chars[start:end])
                    if start > 0:
                        substr = "##" + substr
                    if substr in self.vocab:
                        cur_substr = substr
                        break
                    end -= 1
                if cur_substr is None:
                    is_bad = True
                    break
                sub_tokens.append(cur_substr)
                start = end

            if is_bad:
                output_tokens.append(self.unk_token)
            else:
                output_tokens.extend(sub_tokens)
        return output_tokens
Tokenizes a piece of text into its word pieces.
This uses a greedy longest-match-first algorithm to perform tokenization
using the given vocabulary.

Args:
text: A single token or whitespace separated tokens. This should have
already been passed through `BasicTokenizer`.

Returns:
list (str): A list of wordpiece tokens.

Examples:
.. code-block::

from paddlenlp.transformers import BertTokenizer, WordpieceTokenizer

berttokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
vocab  = berttokenizer.vocab
unk_token = berttokenizer.unk_token

wordpiecetokenizer = WordpieceTokenizer(vocab,unk_token)
inputs = wordpiecetokenizer.tokenize("unaffable")
print(inputs)
ptstructure/vqa/pytorchnlp/transformers/bert/tokenizer.py