

def NextToken(self):
    'Fetch the next token by trying to match any of the regexes in order.'
    if (not self.buffer):
        return
    current_state = self.state
    for token in self._tokens:
        if (token.state_regex and (not token.state_regex.match(current_state))):
            continue
        if self.verbose:
            logging.debug('%s: Trying to match %r with %r', self.state, self.buffer[:10], token.re_str)
        m = token.regex.match(self.buffer)
        if (not m):
            continue
        if self.verbose:
            logging.debug('%s matched %s', token.re_str, m.group(0))
        if (m.end() == 0):
            raise RuntimeError('Lexer bug! Token can not match the empty string.')
        self.processed_buffer += self.buffer[:m.end()]
        self.buffer = self.buffer[m.end():]
        self.processed += m.end()
        next_state = token.next_state
        for action in token.actions:
            if self.verbose:
                logging.debug('Calling %s with %s', action, m.group(0))
            cb = getattr(self, action, self.Default)
            try:
                possible_next_state = cb(string=m.group(0), match=m)
                if (possible_next_state == 'CONTINUE'):
                    continue
                elif possible_next_state:
                    next_state = possible_next_state
            except ParseError as e:
                self.Error(e)
        if next_state:
            self.state = next_state
        return token
    self.Error(('Lexer stuck at state %s' % self.state))
    self.processed_buffer += self.buffer[:1]
    self.buffer = self.buffer[1:]
    return 'Error'

dataset/ETHPy150Open google/grr/grr/lib/lexer.py Lexer.NextToken/original