def tokenize(self):
        """Breaks a stream up into tokens.

        Yields tuples of the form (line_number, column, Tokenizer.TOKEN).
        """
        def readlines(s):
            while True:
                line = s.readline()
                if line != "":
                    yield line.rstrip()
                else:
                    break

        for self.lineno, line in enumerate(readlines(self.stream), 1):
            for self.column, part in self.split(line):
                if part[0] == "#": # COMMENT
                    break
                yield (self.lineno, self.column, self.tokentype(part))
def tokenize ( self ) : def readlines ( s ) : while True : line = s . readline ( ) if line != "" : yield line . rstrip ( ) else : break for self . lineno , line in enumerate ( readlines ( self . stream ) , 1 ) : for self . column , part in self . split ( line ) : if part [ 0 ] == "#" : break yield ( self . lineno , self . column , self . tokentype ( part ) )
Breaks a stream up into tokens.

        Yields tuples of the form (line_number, column, Tokenizer.TOKEN).