

def get_tokens_unprocessed(self, text, stack=('root',)):
    ' Split ``text`` into (tokentype, text) pairs.\n\n        Monkeypatched to store the final stack on the object itself.\n    '
    pos = 0
    tokendefs = self._tokens
    if hasattr(self, '_saved_state_stack'):
        statestack = list(self._saved_state_stack)
    else:
        statestack = list(stack)
    statetokens = tokendefs[statestack[(- 1)]]
    while 1:
        for (rexmatch, action, new_state) in statetokens:
            m = rexmatch(text, pos)
            if m:
                if (type(action) is _TokenType):
                    (yield (pos, action, m.group()))
                else:
                    for item in action(self, m):
                        (yield item)
                pos = m.end()
                if (new_state is not None):
                    if isinstance(new_state, tuple):
                        for state in new_state:
                            if (state == '#pop'):
                                statestack.pop()
                            elif (state == '#push'):
                                statestack.append(statestack[(- 1)])
                            else:
                                statestack.append(state)
                    elif isinstance(new_state, int):
                        del statestack[new_state:]
                    elif (new_state == '#push'):
                        statestack.append(statestack[(- 1)])
                    else:
                        assert False, ('wrong state def: %r' % new_state)
                    statetokens = tokendefs[statestack[(- 1)]]
                break
        else:
            try:
                if (text[pos] == '\n'):
                    pos += 1
                    statestack = ['root']
                    statetokens = tokendefs['root']
                    (yield (pos, Text, '\n'))
                    continue
                (yield (pos, Error, text[pos]))
                pos += 1
            except IndexError:
                break
    self._saved_state_stack = list(statestack)

dataset/ETHPy150Open ipython/ipython-py3k/IPython/frontend/qt/console/pygments_highlighter.py get_tokens_unprocessed/original