Split string `text` into word tokens using the Penn Treebank rules

def word_tokenize(text):
    """
    Split string `text` into word tokens using the Penn Treebank rules
    """
    for (regexp, replacement) in RULES1:
        text = sub(regexp, replacement, text)
    # add extra space to make things easier
    text = " " + text + " "
    for (regexp, replacement) in RULES2:
        text = sub(regexp, replacement, text)
    for regexp in CONTRACTIONS:
        text = sub(regexp, r"\1 \2 ", text)
    # split and return
    return text.split()