def preproc_utts(utts):
    utt_sents = [utt.split(' ') for utt in utts]
    # FIXME PARAM specific to swbd eval2000
    utt_sents = [[word for word in utt if word != '(%hesitation)'] for utt in utt_sents]
    utt_sents = [[word[1:-1] if word.startswith('(') else word for word in utt] for utt in utt_sents]

    # Split up characters while preserving special characters and adding back spaces
    text = [[list(sent[k] + (' ' if k < len(sent) - 1 else ''))
            if sent[k] not in SPECIALS_LIST else [sent[k]]
            for k in xrange(len(sent))] for sent in utt_sents]
    # Flatten nested list
    text = [[c for w in s for c in w] for s in text]
    # Add start and end symbols and replace spaces with space token
    text = [['<s>'] + [c if c != ' ' else SPACE for c in utt] + ['</s>'] for utt in text]
    return text
Convert raw text into character language model format
dataset/ETHPy150Open amaas/stanford-ctc/ctc_fast/clm/prep_text.py/preproc_utts