<s> from __future__ import print_function <EOL> import os <EOL> import copy <EOL> import multiprocessing as multi <EOL> import logging <EOL> import numpy as np <EOL> from collections import defaultdict <EOL> from sklearn . preprocessing . label import LabelBinarizer , MultiLabelBinarizer <EOL> import ipdb <EOL> from marmot . util . simple_corpus import SimpleCorpus <EOL> from marmot . experiment . import_utils import list_of_lists <EOL> logging . basicConfig ( format = '<STR_LIT>' , level = logging . INFO ) <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> def convert_alignments ( align_list , n_words ) : <EOL> new_align = [ [ ] for i in range ( n_words ) ] <EOL> for pair in align_list : <EOL> two_digits = pair . split ( '<STR_LIT:->' ) <EOL> new_align [ int ( two_digits [ <NUM_LIT:1> ] ) ] . append ( int ( two_digits [ <NUM_LIT:0> ] ) ) <EOL> return new_align <EOL> def create_context ( repr_dict , sentence_id = None ) : <EOL> '''<STR_LIT>''' <EOL> context_list = [ ] <EOL> if '<STR_LIT:target>' not in repr_dict : <EOL> print ( "<STR_LIT>" ) <EOL> return [ ] <EOL> if '<STR_LIT>' not in repr_dict : <EOL> print ( "<STR_LIT>" ) <EOL> print ( repr_dict ) <EOL> return [ ] <EOL> active_keys = repr_dict . keys ( ) <EOL> active_keys . remove ( '<STR_LIT>' ) <EOL> for idx , word in enumerate ( repr_dict [ '<STR_LIT:target>' ] ) : <EOL> c = { } <EOL> c [ '<STR_LIT>' ] = word <EOL> c [ '<STR_LIT:index>' ] = idx <EOL> if sentence_id is not None : <EOL> c [ '<STR_LIT>' ] = sentence_id <EOL> if type ( repr_dict [ '<STR_LIT>' ] ) == list or type ( repr_dict [ '<STR_LIT>' ] ) == np . ndarray : <EOL> c [ '<STR_LIT>' ] = repr_dict [ '<STR_LIT>' ] [ idx ] <EOL> c [ '<STR_LIT>' ] = repr_dict [ '<STR_LIT>' ] <EOL> elif type ( repr_dict [ '<STR_LIT>' ] ) == int : <EOL> c [ '<STR_LIT>' ] = repr_dict [ '<STR_LIT>' ] [ idx ] <EOL> else : <EOL> print ( "<STR_LIT>" , type ( repr_dict [ '<STR_LIT>' ] ) ) <EOL> return [ ] <EOL> for k in active_keys : <EOL> c [ k ] = repr_dict [ k ] <EOL> context_list . append ( c ) <EOL> return context_list <EOL> def create_contexts ( data_obj , data_type = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> contexts = [ ] <EOL> if '<STR_LIT:target>' not in data_obj : <EOL> print ( "<STR_LIT>" ) <EOL> return [ ] <EOL> if '<STR_LIT>' not in data_obj : <EOL> print ( "<STR_LIT>" ) <EOL> return [ ] <EOL> for s_idx , sents in enumerate ( zip ( * data_obj . values ( ) ) ) : <EOL> if data_type == '<STR_LIT>' : <EOL> contexts . append ( create_context ( { data_obj . keys ( ) [ i ] : sents [ i ] for i in range ( len ( sents ) ) } , sentence_id = s_idx ) ) <EOL> else : <EOL> contexts . extend ( create_context ( { data_obj . keys ( ) [ i ] : sents [ i ] for i in range ( len ( sents ) ) } , sentence_id = s_idx ) ) <EOL> if data_type == '<STR_LIT>' : <EOL> new_contexts = defaultdict ( list ) <EOL> for cont in contexts : <EOL> new_contexts [ cont [ '<STR_LIT>' ] ] . append ( cont ) <EOL> contexts = copy . deepcopy ( new_contexts ) <EOL> return contexts <EOL> def flatten ( lofl ) : <EOL> if list_of_lists ( lofl ) : <EOL> return [ item for sublist in lofl for item in sublist ] <EOL> elif type ( lofl ) == dict : <EOL> return lofl . values ( ) <EOL> def map_feature_extractor ( ( context , extractor ) ) : <EOL> return extractor . get_features ( context ) <EOL> def contexts_to_features ( contexts , feature_extractors , workers = <NUM_LIT:1> ) : <EOL> if workers == <NUM_LIT:1> : <EOL> return [ [ x for a_list in [ map_feature_extractor ( ( context , extractor ) ) for extractor in feature_extractors ] for x in a_list ] for context in contexts ] <EOL> else : <EOL> res_list = [ ] <EOL> pool = multi . Pool ( workers ) <EOL> logger . info ( '<STR_LIT>' + str ( len ( contexts ) ) + '<STR_LIT>' ) <EOL> for extractor in feature_extractors : <EOL> context_list = [ ( cont , extractor ) for cont in contexts ] <EOL> features = pool . map ( map_feature_extractor , context_list ) <EOL> res_list . append ( features ) <EOL> intermediate = [ [ x [ i ] for x in res_list ] for i in range ( len ( res_list [ <NUM_LIT:0> ] ) ) ] <EOL> res_list = [ flatten ( x ) for x in intermediate ] <EOL> pool . close ( ) <EOL> pool . join ( ) <EOL> return res_list <EOL> def tags_from_contexts ( contexts ) : <EOL> return [ context [ '<STR_LIT>' ] for context in contexts ] <EOL> def fit_binarizers ( all_values ) : <EOL> binarizers = { } <EOL> for f in range ( len ( all_values [ <NUM_LIT:0> ] ) ) : <EOL> cur_features = [ context [ f ] for context in all_values ] <EOL> if type ( cur_features [ <NUM_LIT:0> ] ) == str or type ( cur_features [ <NUM_LIT:0> ] ) == unicode : <EOL> lb = LabelBinarizer ( ) <EOL> lb . fit ( cur_features ) <EOL> binarizers [ f ] = lb <EOL> elif type ( cur_features [ <NUM_LIT:0> ] ) == list : <EOL> mlb = MultiLabelBinarizer ( ) <EOL> cur_features . append ( tuple ( ( "<STR_LIT>" , ) ) ) <EOL> mlb . fit ( [ tuple ( x ) for x in cur_features ] ) <EOL> binarizers [ f ] = mlb <EOL> return binarizers <EOL> def binarize ( features , binarizers ) : <EOL> assert ( list_of_lists ( features ) ) <EOL> num_features = len ( features [ <NUM_LIT:0> ] ) <EOL> if binarizers != { } and max ( binarizers . keys ( ) ) >= num_features : <EOL> print ( "<STR_LIT>" , max ( binarizers . keys ( ) ) ) <EOL> print ( "<STR_LIT>" , num_features ) <EOL> print ( "<STR_LIT>" , features [ <NUM_LIT:0> ] ) <EOL> assert ( binarizers == { } or max ( binarizers . keys ( ) ) < num_features ) <EOL> binarized_cols = [ ] <EOL> for i in range ( num_features ) : <EOL> cur_values = [ f [ i ] for f in features ] <EOL> if i in binarizers : <EOL> binarizer = binarizers [ i ] <EOL> if type ( binarizer ) == LabelBinarizer : <EOL> binarized_cols . append ( binarizer . transform ( cur_values ) ) <EOL> elif type ( binarizer ) == MultiLabelBinarizer : <EOL> assert ( list_of_lists ( cur_values ) ) <EOL> cur_values_default = [ ] <EOL> default_value = binarizer . classes_ [ - <NUM_LIT:1> ] <EOL> for a_list in cur_values : <EOL> new_list = list ( a_list ) <EOL> for j , val in enumerate ( new_list ) : <EOL> if val not in binarizer . classes_ : <EOL> new_list [ j ] = default_value <EOL> cur_values_default . append ( tuple ( new_list ) ) <EOL> transformed = binarizer . transform ( cur_values_default ) <EOL> binarized_cols . append ( transformed ) <EOL> else : <EOL> raise NotImplementedError ( '<STR_LIT>' . format ( type ( binarizer ) ) ) <EOL> else : <EOL> binarized_cols . append ( np . array ( cur_values ) . reshape ( len ( cur_values ) , <NUM_LIT:1> ) ) <EOL> assert ( len ( binarized_cols ) == num_features ) , '<STR_LIT>' <EOL> new_features = np . hstack ( binarized_cols ) <EOL> return new_features </s>
