streamcorpus_pipeline/_tokenizer.py
def make_sentences(self, stream_item):
        'assemble Sentence and Token objects'
        self.make_label_index(stream_item)
        sentences = []
        token_num = 0
        new_mention_id = 0
        for sent_start, sent_end, sent_str in self._sentences(
                stream_item.body.clean_visible):
            assert isinstance(sent_str, unicode)
            sent = Sentence()
            sentence_pos = 0
            for start, end in self.word_tokenizer.span_tokenize(sent_str):
                token_str = sent_str[start:end].encode('utf8')
                tok = Token(
                    token_num=token_num,
                    token=token_str,
                    sentence_pos=sentence_pos,
                )
                tok.offsets[OffsetType.CHARS] = Offset(
                    type=OffsetType.CHARS,
                    first=sent_start + start,
                    length=end - start,
                )
                # whitespace tokenizer will never get a token
                # boundary in the middle of an 'author' label
                try:
                    label = self.label_index.find_le(sent_start + start)
                except ValueError:
                    label = None
                if label:
                    off = label.offsets[OffsetType.CHARS]
                    if off.first + off.length > sent_start + start:
                        streamcorpus.add_annotation(tok, label)
                        logger.debug('adding label to tok: %r has %r',
                                     tok.token, label.target.target_id)

                        if label in self.label_to_mention_id:
                            mention_id = self.label_to_mention_id[label]
                        else:
                            mention_id = new_mention_id
                            new_mention_id += 1
                            self.label_to_mention_id[label] = mention_id

                        tok.mention_id = mention_id

                token_num += 1
                sentence_pos += 1
                sent.tokens.append(tok)
            sentences.append(sent)
        return sentences
def make_sentences(self, stream_item):
        'assemble Sentence and Token objects'
        self.make_label_index(stream_item)
        sentences = []
        token_num = 0
        new_mention_id = 0
        for sent_start, sent_end, sent_str in self._sentences(
                stream_item.body.clean_visible):
            assert isinstance(sent_str, unicode)
            sent = Sentence()
            sentence_pos = 0
            for start, end in self.word_tokenizer.span_tokenize(sent_str):
                token_str = sent_str[start:end].encode('utf8')
                tok = Token(
                    token_num=token_num,
                    token=token_str,
                    sentence_pos=sentence_pos,
                )
                tok.offsets[OffsetType.CHARS] = Offset(
                    type=OffsetType.CHARS,
                    first=sent_start + start,
                    length=end - start,
                )
                # whitespace tokenizer will never get a token
                # boundary in the middle of an 'author' label
                try:
                    label = self.label_index.find_le(sent_start + start)
                except ValueError:
                    label = None
                if label:
                    off = label.offsets[OffsetType.CHARS]
                    if off.first + off.length > sent_start + start:
                        streamcorpus.add_annotation(tok, label)
                        logger.debug('adding label to tok: %r has %r',
                                     tok.token, label.target.target_id)

                        if label in self.label_to_mention_id:
                            mention_id = self.label_to_mention_id[label]
                        else:
                            mention_id = new_mention_id
                            new_mention_id += 1
                            self.label_to_mention_id[label] = mention_id

                        tok.mention_id = mention_id

                token_num += 1
                sentence_pos += 1
                sent.tokens.append(tok)
            sentences.append(sent)
        return sentences
def make_sentences ( self , stream_item ) : self . make_label_index ( stream_item ) sentences = [ ] token_num = 0 new_mention_id = 0 for sent_start , sent_end , sent_str in self . _sentences ( stream_item . body . clean_visible ) : assert isinstance ( sent_str , unicode ) sent = Sentence ( ) sentence_pos = 0 for start , end in self . word_tokenizer . span_tokenize ( sent_str ) : token_str = sent_str [ start : end ] . encode ( 'utf8' ) tok = Token ( token_num = token_num , token = token_str , sentence_pos = sentence_pos , ) tok . offsets [ OffsetType . CHARS ] = Offset ( type = OffsetType . CHARS , first = sent_start + start , length = end - start , ) # whitespace tokenizer will never get a token # boundary in the middle of an 'author' label try : label = self . label_index . find_le ( sent_start + start ) except ValueError : label = None if label : off = label . offsets [ OffsetType . CHARS ] if off . first + off . length > sent_start + start : streamcorpus . add_annotation ( tok , label ) logger . debug ( 'adding label to tok: %r has %r' , tok . token , label . target . target_id ) if label in self . label_to_mention_id : mention_id = self . label_to_mention_id [ label ] else : mention_id = new_mention_id new_mention_id += 1 self . label_to_mention_id [ label ] = mention_id tok . mention_id = mention_id token_num += 1 sentence_pos += 1 sent . tokens . append ( tok ) sentences . append ( sent ) return sentences
assemble Sentence and Token objects
assemble Sentence and Token objects