streamcorpus_pipeline/_run_lingpipe.py
def align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path):
    '''
    iterate through the i_chunk and tmp_ner_path to generate a new
    Chunk with body.ner
    '''
    o_chunk = Chunk()
    input_iter = i_chunk.__iter__()
    ner = ''
    stream_id = None

    all_ner = xml.dom.minidom.parse(open(tmp_ner_path))

    for raw_ner in all_ner.getElementsByTagName('FILENAME'):
        
        stream_item = input_iter.next()
        ## get stream_id out of the XML
        stream_id = raw_ner.attributes.get('docid').value
        assert stream_id and stream_id == stream_item.stream_id, \
            '%s != %s\nner=%r' % (stream_id, stream_item.stream_id, ner)

        tagger_id = 'lingpipe'
        tagging = Tagging()
        tagging.tagger_id = tagger_id
        ## get this one file out of its FILENAME tags
        tagged_doc = list(lingpipe.files(raw_ner.toxml()))[0][1]
        tagging.raw_tagging = tagged_doc
        tagging.generation_time = streamcorpus.make_stream_time()
        stream_item.body.taggings[tagger_id] = tagging

        sentences = list(lingpipe.sentences(tagged_doc))

        ## make JS labels on individual tokens
        assert stream_item.ratings[0].mentions, stream_item.stream_id
        john_smith_label = Label()
        john_smith_label.annotator = stream_item.ratings[0].annotator
        john_smith_label.target_id = stream_item.ratings[0].target_id

        # first map all corefchains to their words
        equiv_ids = collections.defaultdict(lambda: set())
        for sent in sentences:
            for tok in sent.tokens:
                if tok.entity_type is not None:
                    equiv_ids[tok.equiv_id].add(cleanse(tok.token))

        ## find all the chains that are John Smith
        johnsmiths = set()
        for equiv_id, names in equiv_ids.items():
            ## detect 'smith' in 'smithye'
            _names = cleanse(' '.join(names))
            if 'john' in _names and 'smith' in _names:
                johnsmiths.add(equiv_id)

        print len(johnsmiths)
        ## now apply the label
        for sent in sentences:
            for tok in sent.tokens:
                if tok.equiv_id in johnsmiths:
                    tok.labels = [john_smith_label]                

        stream_item.body.sentences[tagger_id] = sentences
        
        o_chunk.add(stream_item)

    ## put the o_chunk bytes into the specified file
    open(tmp_done_path, 'wb').write(str(o_chunk))
    ## replace this with log.info()
    print 'created %s' % tmp_done_path
def align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path):
    '''
    iterate through the i_chunk and tmp_ner_path to generate a new
    Chunk with body.ner
    '''
    o_chunk = Chunk()
    input_iter = i_chunk.__iter__()
    ner = ''
    stream_id = None

    all_ner = xml.dom.minidom.parse(open(tmp_ner_path))

    for raw_ner in all_ner.getElementsByTagName('FILENAME'):
        
        stream_item = input_iter.next()
        ## get stream_id out of the XML
        stream_id = raw_ner.attributes.get('docid').value
        assert stream_id and stream_id == stream_item.stream_id, \
            '%s != %s\nner=%r' % (stream_id, stream_item.stream_id, ner)

        tagger_id = 'lingpipe'
        tagging = Tagging()
        tagging.tagger_id = tagger_id
        ## get this one file out of its FILENAME tags
        tagged_doc = list(lingpipe.files(raw_ner.toxml()))[0][1]
        tagging.raw_tagging = tagged_doc
        tagging.generation_time = streamcorpus.make_stream_time()
        stream_item.body.taggings[tagger_id] = tagging

        sentences = list(lingpipe.sentences(tagged_doc))

        ## make JS labels on individual tokens
        assert stream_item.ratings[0].mentions, stream_item.stream_id
        john_smith_label = Label()
        john_smith_label.annotator = stream_item.ratings[0].annotator
        john_smith_label.target_id = stream_item.ratings[0].target_id

        # first map all corefchains to their words
        equiv_ids = collections.defaultdict(lambda: set())
        for sent in sentences:
            for tok in sent.tokens:
                if tok.entity_type is not None:
                    equiv_ids[tok.equiv_id].add(cleanse(tok.token))

        ## find all the chains that are John Smith
        johnsmiths = set()
        for equiv_id, names in equiv_ids.items():
            ## detect 'smith' in 'smithye'
            _names = cleanse(' '.join(names))
            if 'john' in _names and 'smith' in _names:
                johnsmiths.add(equiv_id)

        print len(johnsmiths)
        ## now apply the label
        for sent in sentences:
            for tok in sent.tokens:
                if tok.equiv_id in johnsmiths:
                    tok.labels = [john_smith_label]                

        stream_item.body.sentences[tagger_id] = sentences
        
        o_chunk.add(stream_item)

    ## put the o_chunk bytes into the specified file
    open(tmp_done_path, 'wb').write(str(o_chunk))
    ## replace this with log.info()
    print 'created %s' % tmp_done_path
def align_chunk_with_ner ( tmp_ner_path , i_chunk , tmp_done_path ) : o_chunk = Chunk ( ) input_iter = i_chunk . __iter__ ( ) ner = '' stream_id = None all_ner = xml . dom . minidom . parse ( open ( tmp_ner_path ) ) for raw_ner in all_ner . getElementsByTagName ( 'FILENAME' ) : stream_item = input_iter . next ( ) ## get stream_id out of the XML stream_id = raw_ner . attributes . get ( 'docid' ) . value assert stream_id and stream_id == stream_item . stream_id , '%s != %s\nner=%r' % ( stream_id , stream_item . stream_id , ner ) tagger_id = 'lingpipe' tagging = Tagging ( ) tagging . tagger_id = tagger_id ## get this one file out of its FILENAME tags tagged_doc = list ( lingpipe . files ( raw_ner . toxml ( ) ) ) [ 0 ] [ 1 ] tagging . raw_tagging = tagged_doc tagging . generation_time = streamcorpus . make_stream_time ( ) stream_item . body . taggings [ tagger_id ] = tagging sentences = list ( lingpipe . sentences ( tagged_doc ) ) ## make JS labels on individual tokens assert stream_item . ratings [ 0 ] . mentions , stream_item . stream_id john_smith_label = Label ( ) john_smith_label . annotator = stream_item . ratings [ 0 ] . annotator john_smith_label . target_id = stream_item . ratings [ 0 ] . target_id # first map all corefchains to their words equiv_ids = collections . defaultdict ( lambda : set ( ) ) for sent in sentences : for tok in sent . tokens : if tok . entity_type is not None : equiv_ids [ tok . equiv_id ] . add ( cleanse ( tok . token ) ) ## find all the chains that are John Smith johnsmiths = set ( ) for equiv_id , names in equiv_ids . items ( ) : ## detect 'smith' in 'smithye' _names = cleanse ( ' ' . join ( names ) ) if 'john' in _names and 'smith' in _names : johnsmiths . add ( equiv_id ) print len ( johnsmiths ) ## now apply the label for sent in sentences : for tok in sent . tokens : if tok . equiv_id in johnsmiths : tok . labels = [ john_smith_label ] stream_item . body . sentences [ tagger_id ] = sentences o_chunk . add ( stream_item ) ## put the o_chunk bytes into the specified file open ( tmp_done_path , 'wb' ) . write ( str ( o_chunk ) ) ## replace this with log.info() print 'created %s' % tmp_done_path
iterate through the i_chunk and tmp_ner_path to generate a new
    Chunk with body.ner
iterate through the i_chunk and tmp_ner_path to generate a new Chunk with body . ner