

def tokenize2(text):
    tokenized = []
    text = text.replace('\n', ' \n ')
    text = text.replace('\t', ' \t ')
    text = text.replace('.', ' . ')
    text = text.replace(',', ' , ')
    text = text.replace(':', ' : ')
    text = text.replace(';', ' ; ')
    text = text.replace('!', ' ! ')
    text = text.replace('?', ' ? ')
    text = text.replace('>', ' > ')
    text = text.replace('<', ' < ')
    text = text.replace('"', ' " ')
    text = text.replace('(', ' ( ')
    text = text.replace(')', ' ) ')
    text = text.replace('[', ' [ ')
    text = text.replace(']', ' ] ')
    text = text.replace("'s", " 's")
    text = text.replace("'t", " 't")
    text = text.replace("'re", " 're")
    text = text.replace("'ve", " 've")
    text = text.replace("'ll", " 'll")
    text = text.replace("'m", " 'm")
    text = text.replace("'d", " 'd")
    text = text.replace(" '", ' `')
    text = text.replace("'", " ' ")
    tokens = text.split(' ')
    tokens = [token for token in tokens if token]
    return tokens

dataset/ETHPy150Open IndicoDataSolutions/Foxhound/foxhound/preprocessing.py tokenize2/original