streamcorpus_pipeline/_tokenizer.py
def _sentences(self, clean_visible):
        'generate strings identified as sentences'
        previous_end = 0
        clean_visible = clean_visible.decode('utf8')
        for start, end in self.sentence_tokenizer.span_tokenize(clean_visible):
            # no need to check start, because the first byte of text
            # is always first byte of first sentence, and we will
            # have already made the previous sentence longer on the
            # end if there was an overlap.
            if start < previous_end:
                start = previous_end
                if start > end:
                    # skip this sentence... because it was eaten by
                    # an earlier sentence with a label
                    continue
            try:
                label = self.label_index.find_le(end)
            except ValueError:
                label = None
            if label:
                ## avoid splitting a label
                off = label.offsets[OffsetType.CHARS]
                end = max(off.first + off.length, end)
            previous_end = end
            sent_str = clean_visible[start:end]
            yield start, end, sent_str
def _sentences(self, clean_visible):
        'generate strings identified as sentences'
        previous_end = 0
        clean_visible = clean_visible.decode('utf8')
        for start, end in self.sentence_tokenizer.span_tokenize(clean_visible):
            # no need to check start, because the first byte of text
            # is always first byte of first sentence, and we will
            # have already made the previous sentence longer on the
            # end if there was an overlap.
            if start < previous_end:
                start = previous_end
                if start > end:
                    # skip this sentence... because it was eaten by
                    # an earlier sentence with a label
                    continue
            try:
                label = self.label_index.find_le(end)
            except ValueError:
                label = None
            if label:
                ## avoid splitting a label
                off = label.offsets[OffsetType.CHARS]
                end = max(off.first + off.length, end)
            previous_end = end
            sent_str = clean_visible[start:end]
            yield start, end, sent_str
def _sentences ( self , clean_visible ) : previous_end = 0 clean_visible = clean_visible . decode ( 'utf8' ) for start , end in self . sentence_tokenizer . span_tokenize ( clean_visible ) : # no need to check start, because the first byte of text # is always first byte of first sentence, and we will # have already made the previous sentence longer on the # end if there was an overlap. if start < previous_end : start = previous_end if start > end : # skip this sentence... because it was eaten by # an earlier sentence with a label continue try : label = self . label_index . find_le ( end ) except ValueError : label = None if label : ## avoid splitting a label off = label . offsets [ OffsetType . CHARS ] end = max ( off . first + off . length , end ) previous_end = end sent_str = clean_visible [ start : end ] yield start , end , sent_str
generate strings identified as sentences
generate strings identified as sentences