<s> """<STR_LIT>""" <EOL> import logging <EOL> from os import makedirs <EOL> from os . path import ( join , <EOL> dirname , <EOL> realpath , <EOL> splitext ) <EOL> from argparse import ( ArgumentParser , <EOL> ArgumentDefaultsHelpFormatter ) <EOL> from src import ( log_dir , <EOL> formatter , <EOL> project_dir ) <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> logging_debug = logging . DEBUG <EOL> logger . setLevel ( logging_debug ) <EOL> loginfo = logger . info <EOL> logdebug = logger . debug <EOL> logerr = logger . error <EOL> logwarn = logger . warning <EOL> sh = logging . StreamHandler ( ) <EOL> sh . setLevel ( logging_debug ) <EOL> sh . setFormatter ( formatter ) <EOL> logger . addHandler ( sh ) <EOL> def main ( ) : <EOL> parser = ArgumentParser ( usage = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> description = '<STR_LIT>' , <EOL> formatter_class = ArgumentDefaultsHelpFormatter ) <EOL> _add_arg = parser . add_argument <EOL> _add_arg ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> type = str , <EOL> required = True ) <EOL> _add_arg ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , <EOL> default = False ) <EOL> _add_arg ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , <EOL> default = False ) <EOL> _add_arg ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , <EOL> default = False ) <EOL> _add_arg ( '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> type = str , <EOL> default = '<STR_LIT:all>' ) <EOL> _add_arg ( '<STR_LIT>' , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> action = '<STR_LIT:store_true>' , <EOL> default = False ) <EOL> _add_arg ( '<STR_LIT>' , '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> type = str , <EOL> default = '<STR_LIT:localhost>' ) <EOL> _add_arg ( '<STR_LIT>' , '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> type = int , <EOL> default = <NUM_LIT> ) <EOL> _add_arg ( '<STR_LIT>' , '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> type = int , <EOL> default = <NUM_LIT:100> ) <EOL> _add_arg ( '<STR_LIT>' , '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> type = str , <EOL> default = join ( log_dir , '<STR_LIT>' ) ) <EOL> args = parser . parse_args ( ) <EOL> from pymongo . errors import ( BulkWriteError , <EOL> ConnectionFailure ) <EOL> from src import ( get_game_files , <EOL> log_format_string ) <EOL> from src . mongodb import ( connect_to_db , <EOL> bulk_extract_features_and_update_db ) <EOL> game_files = args . game_files <EOL> binarize = not args . do_not_binarize_features <EOL> reuse_features = not args . do_not_reuse_extracted_features <EOL> lowercase_text = not args . do_not_lowercase_text <EOL> lowercase_cngrams = args . lowercase_cngrams <EOL> partition = args . partition <EOL> mongodb_host = args . mongodb_host <EOL> mongodb_port = args . mongodb_port <EOL> update_batch_size = args . update_batch_size <EOL> if update_batch_size < <NUM_LIT:1> : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> log_file_path = realpath ( args . log_file_path ) <EOL> log_file_dir = dirname ( log_file_path ) <EOL> if not exists ( log_file_dir ) : <EOL> makedirs ( log_file_dir , exist_ok = True ) <EOL> fh = logging . FileHandler ( log_file_path ) <EOL> fh . setLevel ( logging_debug ) <EOL> fh . setFormatter ( formatter ) <EOL> logger . addHandler ( fh ) <EOL> logdebug ( '<STR_LIT>' . format ( project_dir ) ) <EOL> logdebug ( '<STR_LIT>' . format ( binarize ) ) <EOL> logdebug ( '<STR_LIT>' <EOL> . format ( reuse_features ) ) <EOL> logdebug ( '<STR_LIT>' <EOL> . format ( lowercase_text ) ) <EOL> logdebug ( '<STR_LIT>' <EOL> . format ( lowercase_cngrams ) ) <EOL> logdebug ( '<STR_LIT>' . format ( update_batch_size ) ) <EOL> loginfo ( '<STR_LIT>' <EOL> . format ( mongodb_host , mongodb_port ) ) <EOL> try : <EOL> db = connect_to_db ( host = mongodb_host , port = mongodb_port ) <EOL> except ConnectionFailure as e : <EOL> logerr ( '<STR_LIT>' ) <EOL> logerr ( e ) <EOL> raise e <EOL> db . write_concern [ '<STR_LIT:w>' ] = <NUM_LIT:0> <EOL> game_files = get_game_files ( game_files ) <EOL> for game_file in game_files : <EOL> game = splitext ( game_file ) [ <NUM_LIT:0> ] <EOL> if partition == '<STR_LIT:all>' : <EOL> partition_string = '<STR_LIT>' <EOL> else : <EOL> partition_string = '<STR_LIT>' . format ( partition ) <EOL> loginfo ( '<STR_LIT>' . format ( partition_string , game ) ) <EOL> try : <EOL> updates = bulk_extract_features_and_update_db ( db , <EOL> game , <EOL> partition , <EOL> reuse_nlp_feats = reuse_features , <EOL> use_binarized_nlp_feats = binarize , <EOL> lowercase_text = lowercase_text , <EOL> lowercase_cngrams = lowercase_cngrams , <EOL> update_batch_size = update_batch_size ) <EOL> except BulkWriteError as bwe : <EOL> logerr ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> raise bwe <EOL> if updates : <EOL> loginfo ( '<STR_LIT>' <EOL> . format ( updates ) ) <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
