def word_tokenize(text):
    """
    Split string `text` into word tokens using the Penn Treebank rules
    """
    for (regexp, replacement) in RULES1:
        text = sub(regexp, replacement, text)
    # add extra space to make things easier
    text = " " + text + " "
    for (regexp, replacement) in RULES2:
        text = sub(regexp, replacement, text)
    for regexp in CONTRACTIONS:
        text = sub(regexp, r"\1 \2 ", text)
    # split and return
    return text.split()
def word_tokenize ( text ) : for ( regexp , replacement ) in RULES1 : text = sub ( regexp , replacement , text ) text = " " + text + " " for ( regexp , replacement ) in RULES2 : text = sub ( regexp , replacement , text ) for regexp in CONTRACTIONS : text = sub ( regexp , r"\1 \2 " , text ) return text . split ( )
Split string `text` into word tokens using the Penn Treebank rules