<s> import functools <EOL> from framework . auth import Auth <EOL> from website . archiver import ( <EOL> StatResult , AggregateStatResult , <EOL> ARCHIVER_NETWORK_ERROR , <EOL> ARCHIVER_SIZE_EXCEEDED , <EOL> ARCHIVER_FILE_NOT_FOUND , <EOL> ) <EOL> from website . archiver . model import ArchiveJob <EOL> from website import ( <EOL> mails , <EOL> settings <EOL> ) <EOL> def send_archiver_size_exceeded_mails ( src , user , stat_result ) : <EOL> mails . send_mail ( <EOL> to_addr = settings . SUPPORT_EMAIL , <EOL> mail = mails . ARCHIVE_SIZE_EXCEEDED_DESK , <EOL> user = user , <EOL> src = src , <EOL> stat_result = stat_result <EOL> ) <EOL> mails . send_mail ( <EOL> to_addr = user . username , <EOL> mail = mails . ARCHIVE_SIZE_EXCEEDED_USER , <EOL> user = user , <EOL> src = src , <EOL> can_change_preferences = False , <EOL> mimetype = '<STR_LIT:html>' , <EOL> ) <EOL> def send_archiver_copy_error_mails ( src , user , results ) : <EOL> mails . send_mail ( <EOL> to_addr = settings . SUPPORT_EMAIL , <EOL> mail = mails . ARCHIVE_COPY_ERROR_DESK , <EOL> user = user , <EOL> src = src , <EOL> results = results , <EOL> ) <EOL> mails . send_mail ( <EOL> to_addr = user . username , <EOL> mail = mails . ARCHIVE_COPY_ERROR_USER , <EOL> user = user , <EOL> src = src , <EOL> results = results , <EOL> can_change_preferences = False , <EOL> mimetype = '<STR_LIT:html>' , <EOL> ) <EOL> def send_archiver_file_not_found_mails ( src , user , results ) : <EOL> mails . send_mail ( <EOL> to_addr = settings . SUPPORT_EMAIL , <EOL> mail = mails . ARCHIVE_FILE_NOT_FOUND_DESK , <EOL> user = user , <EOL> src = src , <EOL> results = results , <EOL> ) <EOL> mails . send_mail ( <EOL> to_addr = user . username , <EOL> mail = mails . ARCHIVE_FILE_NOT_FOUND_USER , <EOL> user = user , <EOL> src = src , <EOL> results = results , <EOL> can_change_preferences = False , <EOL> mimetype = '<STR_LIT:html>' , <EOL> ) <EOL> def send_archiver_uncaught_error_mails ( src , user , results ) : <EOL> mails . send_mail ( <EOL> to_addr = settings . SUPPORT_EMAIL , <EOL> mail = mails . ARCHIVE_UNCAUGHT_ERROR_DESK , <EOL> user = user , <EOL> src = src , <EOL> results = results , <EOL> ) <EOL> mails . send_mail ( <EOL> to_addr = user . username , <EOL> mail = mails . ARCHIVE_UNCAUGHT_ERROR_USER , <EOL> user = user , <EOL> src = src , <EOL> results = results , <EOL> can_change_preferences = False , <EOL> mimetype = '<STR_LIT:html>' , <EOL> ) <EOL> def handle_archive_fail ( reason , src , dst , user , result ) : <EOL> if reason == ARCHIVER_NETWORK_ERROR : <EOL> send_archiver_copy_error_mails ( src , user , result ) <EOL> elif reason == ARCHIVER_SIZE_EXCEEDED : <EOL> send_archiver_size_exceeded_mails ( src , user , result ) <EOL> elif reason == ARCHIVER_FILE_NOT_FOUND : <EOL> send_archiver_file_not_found_mails ( src , user , result ) <EOL> else : <EOL> send_archiver_uncaught_error_mails ( src , user , result ) <EOL> dst . root . sanction . forcibly_reject ( ) <EOL> dst . root . sanction . save ( ) <EOL> dst . root . delete_registration_tree ( save = True ) <EOL> def archive_provider_for ( node , user ) : <EOL> """<STR_LIT>""" <EOL> return node . get_addon ( settings . ARCHIVE_PROVIDER ) <EOL> def has_archive_provider ( node , user ) : <EOL> """<STR_LIT>""" <EOL> return node . has_addon ( settings . ARCHIVE_PROVIDER ) <EOL> def link_archive_provider ( node , user ) : <EOL> """<STR_LIT>""" <EOL> addon = node . get_or_add_addon ( settings . ARCHIVE_PROVIDER , auth = Auth ( user ) ) <EOL> addon . on_add ( ) <EOL> node . save ( ) <EOL> def aggregate_file_tree_metadata ( addon_short_name , fileobj_metadata , user ) : <EOL> """<STR_LIT>""" <EOL> disk_usage = fileobj_metadata . get ( '<STR_LIT:size>' ) <EOL> if fileobj_metadata [ '<STR_LIT>' ] == '<STR_LIT:file>' : <EOL> result = StatResult ( <EOL> target_name = fileobj_metadata [ '<STR_LIT:name>' ] , <EOL> target_id = fileobj_metadata [ '<STR_LIT:path>' ] . lstrip ( '<STR_LIT:/>' ) , <EOL> disk_usage = disk_usage or <NUM_LIT:0> , <EOL> ) <EOL> return result <EOL> else : <EOL> return AggregateStatResult ( <EOL> target_id = fileobj_metadata [ '<STR_LIT:path>' ] . lstrip ( '<STR_LIT:/>' ) , <EOL> target_name = fileobj_metadata [ '<STR_LIT:name>' ] , <EOL> targets = [ aggregate_file_tree_metadata ( addon_short_name , child , user ) for child in fileobj_metadata . get ( '<STR_LIT>' , [ ] ) ] , <EOL> ) <EOL> def before_archive ( node , user ) : <EOL> link_archive_provider ( node , user ) <EOL> job = ArchiveJob ( <EOL> src_node = node . registered_from , <EOL> dst_node = node , <EOL> initiator = user <EOL> ) <EOL> job . set_targets ( ) <EOL> def _do_get_file_map ( file_tree ) : <EOL> """<STR_LIT>""" <EOL> file_map = [ ] <EOL> stack = [ file_tree ] <EOL> while len ( stack ) : <EOL> tree_node = stack . pop ( <NUM_LIT:0> ) <EOL> if tree_node [ '<STR_LIT>' ] == '<STR_LIT:file>' : <EOL> file_map . append ( ( tree_node [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] , tree_node ) ) <EOL> else : <EOL> stack = stack + tree_node [ '<STR_LIT>' ] <EOL> return file_map <EOL> def _memoize_get_file_map ( func ) : <EOL> cache = { } <EOL> @ functools . wraps ( func ) <EOL> def wrapper ( node ) : <EOL> if node . _id not in cache : <EOL> osf_storage = node . get_addon ( '<STR_LIT>' ) <EOL> file_tree = osf_storage . _get_file_tree ( user = node . creator ) <EOL> cache [ node . _id ] = _do_get_file_map ( file_tree ) <EOL> return func ( node , cache [ node . _id ] ) <EOL> return wrapper <EOL> @ _memoize_get_file_map <EOL> def get_file_map ( node , file_map ) : <EOL> """<STR_LIT>""" <EOL> for ( key , value ) in file_map : <EOL> yield ( key , value , node . _id ) <EOL> for child in node . nodes_primary : <EOL> for key , value , node_id in get_file_map ( child ) : <EOL> yield ( key , value , node_id ) <EOL> def find_registration_file ( value , node ) : <EOL> from website . models import Node <EOL> orig_sha256 = value [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> orig_name = value [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> orig_node = value [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> file_map = get_file_map ( node ) <EOL> for sha256 , value , node_id in file_map : <EOL> registered_from_id = Node . load ( node_id ) . registered_from . _id <EOL> if sha256 == orig_sha256 and registered_from_id == orig_node and orig_name == value [ '<STR_LIT:name>' ] : <EOL> return value , node_id <EOL> return None , None <EOL> def find_question ( schema , qid ) : <EOL> for page in schema [ '<STR_LIT>' ] : <EOL> questions = { <EOL> q [ '<STR_LIT>' ] : q <EOL> for q in page [ '<STR_LIT>' ] <EOL> } <EOL> if qid in questions : <EOL> return questions [ qid ] <EOL> def find_selected_files ( schema , metadata ) : <EOL> targets = [ ] <EOL> paths = [ ( '<STR_LIT>' , p ) for p in schema . schema [ '<STR_LIT>' ] ] <EOL> while len ( paths ) : <EOL> prefix , path = paths . pop ( <NUM_LIT:0> ) <EOL> if path . get ( '<STR_LIT>' ) : <EOL> paths = paths + [ ( '<STR_LIT>' , q ) for q in path [ '<STR_LIT>' ] ] <EOL> elif path . get ( '<STR_LIT:type>' ) : <EOL> qid = path . get ( '<STR_LIT>' , path . get ( '<STR_LIT:id>' ) ) <EOL> if path [ '<STR_LIT:type>' ] == '<STR_LIT:object>' : <EOL> paths = paths + [ ( '<STR_LIT>' . format ( prefix , qid ) , p ) for p in path [ '<STR_LIT>' ] ] <EOL> elif path [ '<STR_LIT:type>' ] == '<STR_LIT>' : <EOL> targets . append ( '<STR_LIT>' . format ( prefix , qid ) . lstrip ( '<STR_LIT:.>' ) ) <EOL> selected = { } <EOL> for t in targets : <EOL> parts = t . split ( '<STR_LIT:.>' ) <EOL> value = metadata . get ( parts . pop ( <NUM_LIT:0> ) ) <EOL> while value and len ( parts ) : <EOL> value = value . get ( parts . pop ( <NUM_LIT:0> ) ) <EOL> if value : <EOL> selected [ t ] = value <EOL> return selected <EOL> VIEW_FILE_URL_TEMPLATE = '<STR_LIT>' <EOL> class SettableDict ( dict ) : <EOL> def set ( self , path , value ) : <EOL> parts = path . split ( '<STR_LIT:.>' ) <EOL> next_item = self <EOL> key = None <EOL> while len ( parts ) : <EOL> item = next_item <EOL> key = parts . pop ( <NUM_LIT:0> ) <EOL> next_item = item . get ( key , { } ) <EOL> item [ key ] = next_item <EOL> item [ key ] = value <EOL> def migrate_file_metadata ( dst , schema ) : <EOL> metadata = dst . registered_meta [ schema . _id ] <EOL> missing_files = [ ] <EOL> updated_metadata = SettableDict ( ) <EOL> selected_files = find_selected_files ( schema , metadata ) <EOL> for path , selected in selected_files . items ( ) : <EOL> registration_file , node_id = find_registration_file ( <EOL> selected , <EOL> dst <EOL> ) <EOL> if not registration_file : <EOL> missing_files . append ( { <EOL> '<STR_LIT>' : selected [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : find_question ( schema . schema , path [ <NUM_LIT:0> ] ) [ '<STR_LIT:title>' ] <EOL> } ) <EOL> continue <EOL> root = path . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> updated_metadata [ root ] = metadata [ root ] <EOL> updated_metadata . set ( <EOL> '<STR_LIT>' . format ( path ) , <EOL> VIEW_FILE_URL_TEMPLATE . format ( node_id = node_id , path = registration_file [ '<STR_LIT:path>' ] . lstrip ( '<STR_LIT:/>' ) ) <EOL> ) <EOL> if missing_files : <EOL> from website . archiver . tasks import ArchivedFileNotFound <EOL> raise ArchivedFileNotFound ( <EOL> registration = dst , <EOL> missing_files = missing_files <EOL> ) <EOL> metadata . update ( updated_metadata ) <EOL> dst . registered_meta [ schema . _id ] = metadata <EOL> dst . save ( ) </s>
