def tokens(self):
        """Return a list of token Spans for this sentence."""
        spans = self.word_tokenizer.span_tokenize(self.text)
        toks = [Token(
            text=self.text[span[0]:span[1]],
            start=span[0] + self.start,
            end=span[1] + self.start,
            lexicon=self.lexicon
        ) for span in spans]
        return toks
def tokens ( self ) : spans = self . word_tokenizer . span_tokenize ( self . text ) toks = [ Token ( text = self . text [ span [ 0 ] : span [ 1 ] ] , start = span [ 0 ] + self . start , end = span [ 1 ] + self . start , lexicon = self . lexicon ) for span in spans ] return toks
Return a list of token Spans for this sentence.