def _tokenize_words(self, plaintext):
        parastart = False
        for line in plaintext.split('\n'):
            if line.strip():
                line_toks = iter(self._lang_vars.word_tokenize(line))

                yield self._Token(next(line_toks),
                        parastart=parastart, linestart=True)
                parastart = False

                for t in line_toks:
                    yield self._Token(t)
            else:
                parastart = True


    #////////////////////////////////////////////////////////////
    #{ Annotation Procedures
    #////////////////////////////////////////////////////////////
Divide the given text into tokens, using the punkt word
dataset/ETHPy150Open nltk/nltk/nltk/tokenize/punkt.py/PunktBaseClass._tokenize_words