def lex(self, text):
        """ Yield (token_type, str_data) tokens.

        The last token will be (EOF, None) where EOF is the singleton object
        defined in this module.
        """
        for match in self.regex.finditer(text):
            for name, _ in self.lexicon:
                m = match.group(name)
                if m is not None:
                    yield (name, m)
                    break
        yield (EOF, None)
def lex ( self , text ) : for match in self . regex . finditer ( text ) : for name , _ in self . lexicon : m = match . group ( name ) if m is not None : yield ( name , m ) break yield ( EOF , None )
Yield (token_type, str_data) tokens.

        The last token will be (EOF, None) where EOF is the singleton object
        defined in this module.