<s> __author__ = '<STR_LIT>' <EOL> import simplejson as json <EOL> import codecs <EOL> from datetime import datetime <EOL> class Filter ( ) : <EOL> FILTERED_RESOURCES = { } <EOL> FILTERED_EXTENSIONS = [ ] <EOL> def __init__ ( self , json_repo_path , json_filtered_repo_path , filtered_resources_path , filtered_extensions_path , type , logger ) : <EOL> self . JSON_REPO_PATH = json_repo_path <EOL> self . JSON_REPO_FILTERED_PATH = json_filtered_repo_path <EOL> self . FILTERED_RESOURCES_PATH = filtered_resources_path <EOL> self . FILTERED_EXTENSIONS_PATH = filtered_extensions_path <EOL> self . type = type <EOL> self . logger = logger <EOL> def get_filtered_files_to_dict ( self ) : <EOL> filtered_file = codecs . open ( self . FILTERED_RESOURCES_PATH , '<STR_LIT:r>' , '<STR_LIT:utf-8>' ) <EOL> for line in filtered_file : <EOL> splitted_line = line . split ( '<STR_LIT::>' ) <EOL> ref = splitted_line [ <NUM_LIT:0> ] . strip ( ) <EOL> dir = splitted_line [ <NUM_LIT:1> ] . strip ( ) <EOL> file = splitted_line [ <NUM_LIT:2> ] . strip ( ) <EOL> if Filter . FILTERED_RESOURCES . get ( ref ) : <EOL> dir2file_dict = Filter . FILTERED_RESOURCES . get ( ref ) <EOL> if dir == "<STR_LIT:*>" : <EOL> dir2file_dict . update ( { '<STR_LIT:*>' : '<STR_LIT:*>' } ) <EOL> Filter . FILTERED_RESOURCES . update ( { ref : dir2file_dict } ) <EOL> elif dir2file_dict . get ( dir ) : <EOL> files = dir2file_dict . get ( dir ) <EOL> if file == "<STR_LIT:*>" : <EOL> dir2file_dict . update ( { dir : [ '<STR_LIT:*>' ] } ) <EOL> Filter . FILTERED_RESOURCES . update ( { ref : dir2file_dict } ) <EOL> else : <EOL> files . append ( file ) <EOL> dir2file_dict . update ( { dir : files } ) <EOL> Filter . FILTERED_RESOURCES . update ( { ref : dir2file_dict } ) <EOL> else : <EOL> if file == "<STR_LIT:*>" : <EOL> dir2file_dict . update ( { dir : [ '<STR_LIT:*>' ] } ) <EOL> Filter . FILTERED_RESOURCES . update ( { ref : dir2file_dict } ) <EOL> else : <EOL> dir2file_dict . update ( { dir : [ file ] } ) <EOL> Filter . FILTERED_RESOURCES . update ( { ref : dir2file_dict } ) <EOL> else : <EOL> if dir == "<STR_LIT:*>" : <EOL> Filter . FILTERED_RESOURCES . update ( { ref : { '<STR_LIT:*>' : '<STR_LIT:*>' } } ) <EOL> else : <EOL> if file == "<STR_LIT:*>" : <EOL> Filter . FILTERED_RESOURCES . update ( { ref : { dir : [ '<STR_LIT:*>' ] } } ) <EOL> else : <EOL> Filter . FILTERED_RESOURCES . update ( { ref : { dir : [ file ] } } ) <EOL> filtered_file . close ( ) <EOL> def get_filtered_extensions_to_list ( self ) : <EOL> file = codecs . open ( self . FILTERED_EXTENSIONS_PATH , '<STR_LIT:r>' , '<STR_LIT:utf-8>' ) <EOL> for line in file : <EOL> ext = line . strip ( ) <EOL> Filter . FILTERED_EXTENSIONS . append ( ext ) <EOL> file . close ( ) <EOL> def is_filtered ( self , ext , ref , dir , file ) : <EOL> found = False <EOL> if ext in Filter . FILTERED_EXTENSIONS : <EOL> found = True <EOL> else : <EOL> if Filter . FILTERED_RESOURCES . get ( ref ) : <EOL> filtered_dirs_in_ref = Filter . FILTERED_RESOURCES . get ( ref ) <EOL> if filtered_dirs_in_ref . get ( '<STR_LIT:*>' ) : <EOL> found = True <EOL> else : <EOL> if filtered_dirs_in_ref . get ( dir ) : <EOL> filtered_files_in_dir = filtered_dirs_in_ref . get ( dir ) <EOL> if '<STR_LIT:*>' in filtered_files_in_dir : <EOL> found = True <EOL> else : <EOL> if file in filtered_files_in_dir : <EOL> found = True <EOL> return found <EOL> def get_dirs ( self , dirs ) : <EOL> if not dirs : <EOL> dirs . append ( '<STR_LIT>' ) <EOL> return dirs <EOL> def select_files ( self ) : <EOL> repo_json = codecs . open ( self . JSON_REPO_PATH , '<STR_LIT:r>' , '<STR_LIT:utf-8>' ) <EOL> filtered_repo_json = codecs . open ( self . JSON_REPO_FILTERED_PATH , '<STR_LIT:w>' , '<STR_LIT:utf-8>' ) <EOL> for json_line in repo_json : <EOL> json_entry = json . loads ( json_line ) <EOL> ref = json_entry . get ( '<STR_LIT>' ) <EOL> dirs = self . get_dirs ( json_entry . get ( '<STR_LIT>' ) ) <EOL> ext = json_entry . get ( '<STR_LIT>' ) <EOL> file = json_entry . get ( '<STR_LIT:name>' ) <EOL> filtered = True <EOL> for dir in dirs : <EOL> if self . is_filtered ( ext , ref , dir , file ) : <EOL> filtered = False <EOL> break <EOL> if not filtered : <EOL> filtered_repo_json . write ( json . dumps ( json_entry ) + '<STR_LIT:\n>' ) <EOL> else : <EOL> self . logger . info ( "<STR_LIT>" + ref + "<STR_LIT>" + dirs [ <NUM_LIT:0> ] + "<STR_LIT>" + file + "<STR_LIT>" + ext + "<STR_LIT>" ) <EOL> repo_json . close ( ) <EOL> filtered_repo_json . close ( ) <EOL> def reject_files ( self ) : <EOL> repo_json = codecs . open ( self . JSON_REPO_PATH , '<STR_LIT:r>' , '<STR_LIT:utf-8>' ) <EOL> filtered_repo_json = codecs . open ( self . JSON_REPO_FILTERED_PATH , '<STR_LIT:w>' , '<STR_LIT:utf-8>' ) <EOL> for json_line in repo_json : <EOL> json_entry = json . loads ( json_line ) <EOL> ref = json_entry . get ( '<STR_LIT>' ) <EOL> dirs = self . get_dirs ( json_entry . get ( '<STR_LIT>' ) ) <EOL> ext = json_entry . get ( '<STR_LIT>' ) <EOL> file = json_entry . get ( '<STR_LIT:name>' ) <EOL> filtered = False <EOL> for dir in dirs : <EOL> if self . is_filtered ( ext , ref , dir , file ) : <EOL> filtered = True <EOL> break <EOL> if not filtered : <EOL> filtered_repo_json . write ( json . dumps ( json_entry ) + '<STR_LIT:\n>' ) <EOL> else : <EOL> self . logger . info ( "<STR_LIT>" + ref + "<STR_LIT>" + dirs [ <NUM_LIT:0> ] + "<STR_LIT>" + file + "<STR_LIT>" + ext + "<STR_LIT>" ) <EOL> repo_json . close ( ) <EOL> filtered_repo_json . close ( ) <EOL> def filter ( self ) : <EOL> start_time = datetime . now ( ) <EOL> if self . FILTERED_EXTENSIONS_PATH : <EOL> self . get_filtered_extensions_to_list ( ) <EOL> if self . FILTERED_RESOURCES_PATH : <EOL> self . get_filtered_files_to_dict ( ) <EOL> if self . type == "<STR_LIT>" : <EOL> self . select_files ( ) <EOL> elif self . type == "<STR_LIT>" : <EOL> self . reject_files ( ) <EOL> else : <EOL> self . logger . error ( "<STR_LIT>" + self . type ) <EOL> end_time = datetime . now ( ) <EOL> minutes_and_seconds = divmod ( ( end_time - start_time ) . total_seconds ( ) , <NUM_LIT> ) <EOL> self . logger . info ( "<STR_LIT>" + str ( minutes_and_seconds [ <NUM_LIT:0> ] ) <EOL> + "<STR_LIT>" + str ( round ( minutes_and_seconds [ <NUM_LIT:1> ] , <NUM_LIT:1> ) ) + "<STR_LIT>" ) </s>
