def _get_tokenizer(__cache=[]):
    if not __cache:
        regex = [x.format(**macros) for x in en_regex]
        regex = u"|".join(regex)
        tokenizer = RegexpTokenizer(regex, flags=re.UNICODE |
                                                 re.MULTILINE |
                                                 re.DOTALL | re.I)
        __cache.append(tokenizer)
    return __cache[0]
Get a tokenizer for english.
dataset/ETHPy150Open machinalis/iepy/iepy/preprocess/tokenizer.py/_get_tokenizer