def tokenizer(text):
    """A lexical analyzer for the `CTfile` formatted files.

    :param str text: `CTfile` formatted text.
    :return: Tuples of data.
    :rtype: py:class:`~collections.namedtuple`
    """
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
def tokenizer ( text ) : for entry in text . split ( '$$$$\n' ) : if entry . rstrip ( ) : lines_stream = deque ( entry . split ( '\n' ) ) else : continue for token in _molfile ( stream = lines_stream ) : yield token if len ( lines_stream ) : for token in _sdfile ( stream = lines_stream ) : yield token yield EndOfFile ( )
A lexical analyzer for the `CTfile` formatted files.

    :param str text: `CTfile` formatted text.
    :return: Tuples of data.
    :rtype: py:class:`~collections.namedtuple`