def tokenize(text: str) -> List:
    """
    Use SimpleTokenizer to tokenize text.
    Args:
        text (`str`):
            Text to tokenize

    Returns:
        Tokenized infos.
    """
    tokenizer = SimpleTokenizer()
    tokens = []
    text = whitespace_clean(basic_clean(text)).lower()
    for token in re.findall(tokenizer.pat, text):
        token = "".join(tokenizer.byte_encoder[b] for b in token.encode("utf-8"))
        tokens.extend(bpe_token for bpe_token in tokenizer.bpe(token).split(" "))
    return tokens
Use SimpleTokenizer to tokenize text.
Args:
text (`str`):
Text to tokenize

Returns:
Tokenized infos.
towhee/models/clip4clip/utils.py