Breaks a stream up into tokens.

        Yields tuples of the form (line_number, column, Tokenizer.TOKEN).

def tokenize(self):
        """Breaks a stream up into tokens.

        Yields tuples of the form (line_number, column, Tokenizer.TOKEN).
        """
        def readlines(s):
            while True:
                line = s.readline()
                if line != "":
                    yield line.rstrip()
                else:
                    break

        for self.lineno, line in enumerate(readlines(self.stream), 1):
            for self.column, part in self.split(line):
                if part[0] == "#": # COMMENT
                    break
                yield (self.lineno, self.column, self.tokentype(part))