def tokenize_middle_english_words(text):
    """Tokenizes ME text:

    >>> tokenize_middle_english_words("And then,   went   I  fastyr!")
    ['And', 'then', ',', 'went', 'I', 'fastyr', '!']

    """

    assert isinstance(text, str)

    text = re.sub(r'\n', r' ', text)
    text = re.sub(r'(?<=.)(?=[\.\";\,\:\-\[\]\(\)!&?])',r' ', text)
    text = re.sub(r'(?<=[\.\";\-\,\:\[\]\(\)!&?])(?=.)',r' ', text)
    text = re.sub(r'\s+',r' ', text)
    text = str.split(text)

    return text
def tokenize_middle_english_words ( text ) : assert isinstance ( text , str ) text = re . sub ( r'\n' , r' ' , text ) text = re . sub ( r'(?<=.)(?=[\.\";\,\:\-\[\]\(\)!&?])' , r' ' , text ) text = re . sub ( r'(?<=[\.\";\-\,\:\[\]\(\)!&?])(?=.)' , r' ' , text ) text = re . sub ( r'\s+' , r' ' , text ) text = str . split ( text ) return text
Tokenizes ME text:

    >>> tokenize_middle_english_words("And then,   went   I  fastyr!")
    ['And', 'then', ',', 'went', 'I', 'fastyr', '!']