<s> """<STR_LIT>""" <EOL> import logging <EOL> LOGGER = logging . getLogger ( '<STR_LIT>' ) <EOL> from bigml . util import invert_dictionary , DEFAULT_LOCALE <EOL> from bigml . fields import DEFAULT_MISSING_TOKENS <EOL> def check_model_structure ( model ) : <EOL> """<STR_LIT>""" <EOL> return ( isinstance ( model , dict ) and '<STR_LIT>' in model and <EOL> model [ '<STR_LIT>' ] is not None and <EOL> ( '<STR_LIT:object>' in model and '<STR_LIT>' in model [ '<STR_LIT:object>' ] or <EOL> '<STR_LIT>' in model ) ) <EOL> class ModelFields ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , fields , objective_id = None , data_locale = None , <EOL> missing_tokens = None ) : <EOL> if isinstance ( fields , dict ) : <EOL> try : <EOL> self . objective_id = objective_id <EOL> self . uniquify_varnames ( fields ) <EOL> self . inverted_fields = invert_dictionary ( fields ) <EOL> self . fields = { } <EOL> self . fields . update ( fields ) <EOL> self . data_locale = data_locale <EOL> self . missing_tokens = missing_tokens <EOL> if self . data_locale is None : <EOL> self . data_locale = DEFAULT_LOCALE <EOL> if self . missing_tokens is None : <EOL> self . missing_tokens = DEFAULT_MISSING_TOKENS <EOL> except KeyError : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> def uniquify_varnames ( self , fields ) : <EOL> """<STR_LIT>""" <EOL> unique_names = set ( [ fields [ key ] [ '<STR_LIT:name>' ] for key in fields ] ) <EOL> if len ( unique_names ) < len ( fields ) : <EOL> self . transform_repeated_names ( fields ) <EOL> def transform_repeated_names ( self , fields ) : <EOL> """<STR_LIT>""" <EOL> if self . objective_id : <EOL> unique_names = [ fields [ self . objective_id ] [ '<STR_LIT:name>' ] ] <EOL> else : <EOL> unique_names = [ ] <EOL> field_ids = sorted ( [ field_id for field_id in fields <EOL> if field_id != self . objective_id ] ) <EOL> for field_id in field_ids : <EOL> new_name = fields [ field_id ] [ '<STR_LIT:name>' ] <EOL> if new_name in unique_names : <EOL> new_name = "<STR_LIT>" . format ( fields [ field_id ] [ '<STR_LIT:name>' ] , <EOL> fields [ field_id ] [ '<STR_LIT>' ] ) <EOL> if new_name in unique_names : <EOL> new_name = "<STR_LIT>" . format ( new_name , field_id ) <EOL> fields [ field_id ] [ '<STR_LIT:name>' ] = new_name <EOL> unique_names . append ( new_name ) <EOL> def normalize ( self , value ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( value , basestring ) and not isinstance ( value , unicode ) : <EOL> value = unicode ( value , "<STR_LIT:utf-8>" ) <EOL> return None if value in self . missing_tokens else value <EOL> def filter_input_data ( self , input_data , by_name = True ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( input_data , dict ) : <EOL> for key , value in input_data . items ( ) : <EOL> value = self . normalize ( value ) <EOL> if value is None : <EOL> del input_data [ key ] <EOL> if by_name : <EOL> input_data = dict ( <EOL> [ [ self . inverted_fields [ key ] , value ] <EOL> for key , value in input_data . items ( ) <EOL> if key in self . inverted_fields and <EOL> ( self . objective_id is None or <EOL> self . inverted_fields [ key ] != self . objective_id ) ] ) <EOL> else : <EOL> input_data = dict ( <EOL> [ [ key , value ] <EOL> for key , value in input_data . items ( ) <EOL> if key in self . fields and <EOL> ( self . objective_id is None or <EOL> key != self . objective_id ) ] ) <EOL> return input_data <EOL> else : <EOL> LOGGER . error ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> return { } </s>
