def parse(self, contents):
        """Parse the document.

        :param contents: The text contents of the document.
        :rtype: a *generator* of tokenized text.
        """
        i = 0
        for text in contents.split(self.delim):
            if not len(text.strip()):
                continue
            words = text.split()
            char_offsets = [0] + [
                int(_) for _ in np.cumsum([len(x) + 1 for x in words])[:-1]
            ]
            text = " ".join(words)
            yield {
                "text": text,
                "words": words,
                "pos_tags": [""] * len(words),
                "ner_tags": [""] * len(words),
                "lemmas": [""] * len(words),
                "dep_parents": [0] * len(words),
                "dep_labels": [""] * len(words),
                "char_offsets": char_offsets,
                "abs_char_offsets": char_offsets,
            }
            i += 1
def parse ( self , contents ) : i = 0 for text in contents . split ( self . delim ) : if not len ( text . strip ( ) ) : continue words = text . split ( ) char_offsets = [ 0 ] + [ int ( _ ) for _ in np . cumsum ( [ len ( x ) + 1 for x in words ] ) [ : - 1 ] ] text = " " . join ( words ) yield { "text" : text , "words" : words , "pos_tags" : [ "" ] * len ( words ) , "ner_tags" : [ "" ] * len ( words ) , "lemmas" : [ "" ] * len ( words ) , "dep_parents" : [ 0 ] * len ( words ) , "dep_labels" : [ "" ] * len ( words ) , "char_offsets" : char_offsets , "abs_char_offsets" : char_offsets , } i += 1
Parse the document.

        :param contents: The text contents of the document.
        :rtype: a *generator* of tokenized text.