def lex(self, text):
        end = len(text)
        state = self.state
        regexes = self.regexes
        toks = self.toks
        start = 0

        while start < end:
            for match in regexes[state].finditer(text, start):
                name = match.lastgroup
                tok = toks[name]
                toktext = match.group(name)
                start += len(toktext)
                yield (tok.name, toktext)

                if tok.next:
                    state = tok.next
                    break

        self.state = state
Lexically analyze `text`.
dataset/ETHPy150Open AppScale/appscale/AppServer/lib/django-1.4/django/utils/jslex.py/Lexer.lex