<gh_stars>1-10
# from tokenizers import (
#     ByteLevelBPETokenizer,
#     CharBPETokenizer,
# )

from tools.word_piece import WordPieceTokenizer
from tools.BBPE import ByteLevelBPETokenizer
from tools.CBPE import CharBPETokenizer


class Tokenizer:
    def __init__(self):
        pass


if __name__ == '__main__':
    pass

# from tokenizers import (
#     ByteLevelBPETokenizer,
#     CharBPETokenizer,
# )

from tools.word_piece import WordPieceTokenizer
from tools.BBPE import ByteLevelBPETokenizer
from tools.CBPE import CharBPETokenizer


class Tokenizer:
    def __init__(self):
        pass


if __name__ == '__main__':
    pass
# from tokenizers import ( #     ByteLevelBPETokenizer, #     CharBPETokenizer, # )