streamcorpus_pipeline/_taggers.py
def multi_token_match(stream_item, aligner_data):
    '''
    iterate through tokens looking for near-exact matches to strings
    in si.ratings...mentions
    '''    
    tagger_id = _get_tagger_id(stream_item, aligner_data)
    sentences = stream_item.body.sentences.get(tagger_id)
    if not sentences:
        return
    ## construct a list of tuples, where the first part of each tuple
    ## is a tuple of cleansed strings, and the second part is the
    ## Token object from which it came.
    tokens = map(lambda tok: (cleanse(tok.token.decode('utf8')).split(' '), tok), 
                 itertools.chain(*[sent.tokens for sent in sentences]))    
    required_annotator_id = aligner_data['annotator_id']
    for annotator_id, ratings in stream_item.ratings.items():
        if (required_annotator_id is None) or (annotator_id == required_annotator_id):
            for rating in ratings:
                label = Label(annotator=rating.annotator,
                              target=rating.target)
                
                num_tokens_matched = 0
                for tok in look_ahead_match(rating, tokens):
                    if aligner_data.get('update_labels'):
                        tok.labels.pop(annotator_id, None)
                    add_annotation(tok, label)
                    num_tokens_matched += 1

                if num_tokens_matched == 0:
                    logger.warning('multi_token_match didn\'t actually match '
                                   'entity %r in stream_id %r',
                                   rating.target.target_id,
                                   stream_item.stream_id)
                else:
                    logger.debug('matched %d tokens for %r in %r',
                                 num_tokens_matched, rating.target.target_id,
                                 stream_item.stream_id)
def multi_token_match(stream_item, aligner_data):
    '''
    iterate through tokens looking for near-exact matches to strings
    in si.ratings...mentions
    '''    
    tagger_id = _get_tagger_id(stream_item, aligner_data)
    sentences = stream_item.body.sentences.get(tagger_id)
    if not sentences:
        return
    ## construct a list of tuples, where the first part of each tuple
    ## is a tuple of cleansed strings, and the second part is the
    ## Token object from which it came.
    tokens = map(lambda tok: (cleanse(tok.token.decode('utf8')).split(' '), tok), 
                 itertools.chain(*[sent.tokens for sent in sentences]))    
    required_annotator_id = aligner_data['annotator_id']
    for annotator_id, ratings in stream_item.ratings.items():
        if (required_annotator_id is None) or (annotator_id == required_annotator_id):
            for rating in ratings:
                label = Label(annotator=rating.annotator,
                              target=rating.target)
                
                num_tokens_matched = 0
                for tok in look_ahead_match(rating, tokens):
                    if aligner_data.get('update_labels'):
                        tok.labels.pop(annotator_id, None)
                    add_annotation(tok, label)
                    num_tokens_matched += 1

                if num_tokens_matched == 0:
                    logger.warning('multi_token_match didn\'t actually match '
                                   'entity %r in stream_id %r',
                                   rating.target.target_id,
                                   stream_item.stream_id)
                else:
                    logger.debug('matched %d tokens for %r in %r',
                                 num_tokens_matched, rating.target.target_id,
                                 stream_item.stream_id)
def multi_token_match ( stream_item , aligner_data ) : tagger_id = _get_tagger_id ( stream_item , aligner_data ) sentences = stream_item . body . sentences . get ( tagger_id ) if not sentences : return ## construct a list of tuples, where the first part of each tuple ## is a tuple of cleansed strings, and the second part is the ## Token object from which it came. tokens = map ( lambda tok : ( cleanse ( tok . token . decode ( 'utf8' ) ) . split ( ' ' ) , tok ) , itertools . chain ( * [ sent . tokens for sent in sentences ] ) ) required_annotator_id = aligner_data [ 'annotator_id' ] for annotator_id , ratings in stream_item . ratings . items ( ) : if ( required_annotator_id is None ) or ( annotator_id == required_annotator_id ) : for rating in ratings : label = Label ( annotator = rating . annotator , target = rating . target ) num_tokens_matched = 0 for tok in look_ahead_match ( rating , tokens ) : if aligner_data . get ( 'update_labels' ) : tok . labels . pop ( annotator_id , None ) add_annotation ( tok , label ) num_tokens_matched += 1 if num_tokens_matched == 0 : logger . warning ( 'multi_token_match didn\'t actually match ' 'entity %r in stream_id %r' , rating . target . target_id , stream_item . stream_id ) else : logger . debug ( 'matched %d tokens for %r in %r' , num_tokens_matched , rating . target . target_id , stream_item . stream_id )
iterate through tokens looking for near-exact matches to strings
    in si.ratings...mentions
iterate through tokens looking for near - exact matches to strings in si . ratings ... mentions