streamcorpus_pipeline/_lingpipe.py
def tokens(self, sentence_dom):
        '''
        Tokenize all the words and preserve NER labels from ENAMEX tags
        '''
        ## keep track of sentence position, which is reset for each
        ## sentence, and used above in _make_token
        self.sent_pos = 0
    
        ## keep track of mention_id, so we can distinguish adjacent
        ## multi-token mentions within the same coref chain
        mention_id = 0

        while len(sentence_dom.childNodes) > 0:
            ## shrink the sentence_dom's child nodes.  In v0_2_0 this
            ## was required to cope with HitMaxi16.  Now it is just to
            ## save memory.
            node = sentence_dom.childNodes.pop(0)

            if node.nodeType == node.TEXT_NODE:
                ## process portion before an ENAMEX tag
                for line in node.data.splitlines(True):
                    self._input_string = line
                    for start, end in self.word_tokenizer.span_tokenize(line):
                        tok = self._make_token(start, end)
                        if tok:
                            yield tok

                    if line.endswith('\n'):
                        ## maintain the index to the current line
                        self.line_idx += 1

                    ## increment index pasat the 'before' portion
                    self.byte_idx += len(line.encode('utf-8'))

            else:
                ## process text inside an ENAMEX tag
                assert node.nodeName == 'ENAMEX', node.nodeName
                chain_id = node.attributes.get('ID').value
                entity_type = node.attributes.get('TYPE').value
                for node in node.childNodes:
                    assert node.nodeType == node.TEXT_NODE, node.nodeType
                    for line in node.data.splitlines(True):
                        self._input_string = line
                        for start, end in self.word_tokenizer.span_tokenize(line):
                            tok = self._make_token(start, end)
                            if tok:
                                if entity_type in _PRONOUNS:
                                    tok.mention_type = MentionType.PRO
                                    tok.entity_type = _ENTITY_TYPES[entity_type]
                                    
                                    ## create an attribute
                                    attr = Attribute(
                                        attribute_type=AttributeType.PER_GENDER,
                                        value=str(_PRONOUNS[entity_type])
                                        )
                                    self.attributes.append(attr)

                                else:
                                    ## regular entity_type
                                    tok.mention_type = MentionType.NAME
                                    tok.entity_type = _ENTITY_TYPES[entity_type]

                                tok.equiv_id = int(chain_id)
                                tok.mention_id = mention_id
                                yield tok

                        if line.endswith('\n'):
                            ## maintain the index to the current line
                            self.line_idx += 1

                        ## increment index pasat the 'before' portion
                        self.byte_idx += len(line.encode('utf-8'))

                ## increment mention_id within this sentence
                mention_id += 1
def tokens(self, sentence_dom):
        '''
        Tokenize all the words and preserve NER labels from ENAMEX tags
        '''
        ## keep track of sentence position, which is reset for each
        ## sentence, and used above in _make_token
        self.sent_pos = 0
    
        ## keep track of mention_id, so we can distinguish adjacent
        ## multi-token mentions within the same coref chain
        mention_id = 0

        while len(sentence_dom.childNodes) > 0:
            ## shrink the sentence_dom's child nodes.  In v0_2_0 this
            ## was required to cope with HitMaxi16.  Now it is just to
            ## save memory.
            node = sentence_dom.childNodes.pop(0)

            if node.nodeType == node.TEXT_NODE:
                ## process portion before an ENAMEX tag
                for line in node.data.splitlines(True):
                    self._input_string = line
                    for start, end in self.word_tokenizer.span_tokenize(line):
                        tok = self._make_token(start, end)
                        if tok:
                            yield tok

                    if line.endswith('\n'):
                        ## maintain the index to the current line
                        self.line_idx += 1

                    ## increment index pasat the 'before' portion
                    self.byte_idx += len(line.encode('utf-8'))

            else:
                ## process text inside an ENAMEX tag
                assert node.nodeName == 'ENAMEX', node.nodeName
                chain_id = node.attributes.get('ID').value
                entity_type = node.attributes.get('TYPE').value
                for node in node.childNodes:
                    assert node.nodeType == node.TEXT_NODE, node.nodeType
                    for line in node.data.splitlines(True):
                        self._input_string = line
                        for start, end in self.word_tokenizer.span_tokenize(line):
                            tok = self._make_token(start, end)
                            if tok:
                                if entity_type in _PRONOUNS:
                                    tok.mention_type = MentionType.PRO
                                    tok.entity_type = _ENTITY_TYPES[entity_type]
                                    
                                    ## create an attribute
                                    attr = Attribute(
                                        attribute_type=AttributeType.PER_GENDER,
                                        value=str(_PRONOUNS[entity_type])
                                        )
                                    self.attributes.append(attr)

                                else:
                                    ## regular entity_type
                                    tok.mention_type = MentionType.NAME
                                    tok.entity_type = _ENTITY_TYPES[entity_type]

                                tok.equiv_id = int(chain_id)
                                tok.mention_id = mention_id
                                yield tok

                        if line.endswith('\n'):
                            ## maintain the index to the current line
                            self.line_idx += 1

                        ## increment index pasat the 'before' portion
                        self.byte_idx += len(line.encode('utf-8'))

                ## increment mention_id within this sentence
                mention_id += 1
def tokens ( self , sentence_dom ) : ## keep track of sentence position, which is reset for each ## sentence, and used above in _make_token self . sent_pos = 0 ## keep track of mention_id, so we can distinguish adjacent ## multi-token mentions within the same coref chain mention_id = 0 while len ( sentence_dom . childNodes ) > 0 : ## shrink the sentence_dom's child nodes.  In v0_2_0 this ## was required to cope with HitMaxi16.  Now it is just to ## save memory. node = sentence_dom . childNodes . pop ( 0 ) if node . nodeType == node . TEXT_NODE : ## process portion before an ENAMEX tag for line in node . data . splitlines ( True ) : self . _input_string = line for start , end in self . word_tokenizer . span_tokenize ( line ) : tok = self . _make_token ( start , end ) if tok : yield tok if line . endswith ( '\n' ) : ## maintain the index to the current line self . line_idx += 1 ## increment index pasat the 'before' portion self . byte_idx += len ( line . encode ( 'utf-8' ) ) else : ## process text inside an ENAMEX tag assert node . nodeName == 'ENAMEX' , node . nodeName chain_id = node . attributes . get ( 'ID' ) . value entity_type = node . attributes . get ( 'TYPE' ) . value for node in node . childNodes : assert node . nodeType == node . TEXT_NODE , node . nodeType for line in node . data . splitlines ( True ) : self . _input_string = line for start , end in self . word_tokenizer . span_tokenize ( line ) : tok = self . _make_token ( start , end ) if tok : if entity_type in _PRONOUNS : tok . mention_type = MentionType . PRO tok . entity_type = _ENTITY_TYPES [ entity_type ] ## create an attribute attr = Attribute ( attribute_type = AttributeType . PER_GENDER , value = str ( _PRONOUNS [ entity_type ] ) ) self . attributes . append ( attr ) else : ## regular entity_type tok . mention_type = MentionType . NAME tok . entity_type = _ENTITY_TYPES [ entity_type ] tok . equiv_id = int ( chain_id ) tok . mention_id = mention_id yield tok if line . endswith ( '\n' ) : ## maintain the index to the current line self . line_idx += 1 ## increment index pasat the 'before' portion self . byte_idx += len ( line . encode ( 'utf-8' ) ) ## increment mention_id within this sentence mention_id += 1
Tokenize all the words and preserve NER labels from ENAMEX tags
Tokenize all the words and preserve NER labels from ENAMEX tags