def process(self, data):
        out = []
        mapping = {}
        tokenTrie = Trie(self.get_all_symbols())
        token = ''
        data_idx = 0
        token_start_idx = data_idx
        while data_idx < len(data):
            cur_data_idx = data_idx
            quoted_token, data_idx = self.process_quoted_token(data_idx, data)
            if quoted_token:
                if not token:
                    token_start_idx = cur_data_idx
                token += quoted_token
                continue

            st = tokenTrie
            c = data[data_idx]
            symbol = ''
            while c in st:
                symbol += c
                st = st[c]
                if len(data)-data_idx > len(symbol):
                    c = data[data_idx+len(symbol)]
                else:
                    break
            if Trie.LEAF in st:
                #token is a complete symbol
                if token:
                    mapping[len(out)] = token_start_idx
                    out.append(token)
                    token = ''
                mapping[len(out)] = data_idx
                out.append(symbol)
                data_idx += len(symbol)
            else:
                if data[data_idx] in ' \t\n': #any whitespace
                    if token:
                        mapping[len(out)] = token_start_idx
                        out.append(token)
                        token = ''
                else:
                    if not token:
                        token_start_idx = data_idx
                    token += data[data_idx]
                data_idx += 1
        if token:
            mapping[len(out)] = token_start_idx
            out.append(token)
        mapping[len(out)] = len(data)
        mapping[len(out)+1] = len(data)+1
        return out, mapping
Split the data into tokens
dataset/ETHPy150Open nltk/nltk/nltk/sem/logic.py/LogicParser.process