

def tokenize(text):
    tokenized = []
    for c in punc:
        text = text.replace(c, ((' ' + c) + ' '))
    for c in contractions:
        text = text.replace(c, (' |' + c))
    text = tokens.replace(" |'", ' `')
    text = text.replace("'", " ' ")
    tokens = text.split(' ')
    tokens = [token for token in tokens if token]
    return tokens

dataset/ETHPy150Open IndicoDataSolutions/Foxhound/foxhound/preprocessing.py tokenize/VarMisuse@67/79 `text`->`tokens`