<s> import requests <EOL> import json <EOL> import httplib as http <EOL> import celery <EOL> from celery . utils . log import get_task_logger <EOL> from modularodm import Q <EOL> from framework . celery_tasks import app as celery_app <EOL> from framework . celery_tasks . utils import logged <EOL> from framework . exceptions import HTTPError <EOL> from website . archiver import ( <EOL> ARCHIVER_SUCCESS , <EOL> ARCHIVER_FAILURE , <EOL> ARCHIVER_SIZE_EXCEEDED , <EOL> ARCHIVER_NETWORK_ERROR , <EOL> ARCHIVER_FILE_NOT_FOUND , <EOL> ARCHIVER_UNCAUGHT_ERROR , <EOL> NO_ARCHIVE_LIMIT , <EOL> AggregateStatResult , <EOL> ) <EOL> from website . archiver import utils <EOL> from website . archiver . model import ArchiveJob <EOL> from website . archiver import signals as archiver_signals <EOL> from website . project import signals as project_signals <EOL> from website . project . model import Node , DraftRegistration <EOL> from website import settings <EOL> from website . app import init_addons , do_set_backends <EOL> def create_app_context ( ) : <EOL> try : <EOL> init_addons ( settings ) <EOL> do_set_backends ( settings ) <EOL> except AssertionError : <EOL> pass <EOL> logger = get_task_logger ( __name__ ) <EOL> class ArchiverSizeExceeded ( Exception ) : <EOL> def __init__ ( self , result , * args , ** kwargs ) : <EOL> super ( ArchiverSizeExceeded , self ) . __init__ ( * args , ** kwargs ) <EOL> self . result = result <EOL> class ArchiverStateError ( Exception ) : <EOL> def __init__ ( self , info , * args , ** kwargs ) : <EOL> super ( ArchiverStateError , self ) . __init__ ( * args , ** kwargs ) <EOL> self . info = info <EOL> class ArchivedFileNotFound ( Exception ) : <EOL> def __init__ ( self , registration , missing_files , * args , ** kwargs ) : <EOL> super ( ArchivedFileNotFound , self ) . __init__ ( * args , ** kwargs ) <EOL> self . draft_registration = DraftRegistration . find_one ( <EOL> Q ( '<STR_LIT>' , '<STR_LIT>' , registration ) <EOL> ) <EOL> self . missing_files = missing_files <EOL> class ArchiverTask ( celery . Task ) : <EOL> abstract = True <EOL> max_retries = <NUM_LIT:0> <EOL> def on_failure ( self , exc , task_id , args , kwargs , einfo ) : <EOL> job = ArchiveJob . load ( kwargs . get ( '<STR_LIT>' ) ) <EOL> if not job : <EOL> raise ArchiverStateError ( { <EOL> '<STR_LIT>' : exc , <EOL> '<STR_LIT:args>' : args , <EOL> '<STR_LIT>' : kwargs , <EOL> '<STR_LIT>' : einfo , <EOL> } ) <EOL> if job . status == ARCHIVER_FAILURE : <EOL> return <EOL> src , dst , user = job . info ( ) <EOL> errors = [ ] <EOL> if isinstance ( exc , ArchiverSizeExceeded ) : <EOL> dst . archive_status = ARCHIVER_SIZE_EXCEEDED <EOL> errors = exc . result <EOL> elif isinstance ( exc , HTTPError ) : <EOL> dst . archive_status = ARCHIVER_NETWORK_ERROR <EOL> errors = [ <EOL> each for each in <EOL> dst . archive_job . target_info ( ) <EOL> if each is not None <EOL> ] <EOL> elif isinstance ( exc , ArchivedFileNotFound ) : <EOL> dst . archive_status = ARCHIVER_FILE_NOT_FOUND <EOL> errors = { <EOL> '<STR_LIT>' : exc . missing_files , <EOL> '<STR_LIT>' : exc . draft_registration <EOL> } <EOL> else : <EOL> dst . archive_status = ARCHIVER_UNCAUGHT_ERROR <EOL> errors = [ einfo ] if einfo else [ ] <EOL> dst . save ( ) <EOL> archiver_signals . archive_fail . send ( dst , errors = errors ) <EOL> @ celery_app . task ( base = ArchiverTask , name = "<STR_LIT>" ) <EOL> @ logged ( '<STR_LIT>' ) <EOL> def stat_addon ( addon_short_name , job_pk ) : <EOL> """<STR_LIT>""" <EOL> addon_name = addon_short_name <EOL> version = None <EOL> if '<STR_LIT>' in addon_short_name : <EOL> addon_name = '<STR_LIT>' <EOL> version = '<STR_LIT>' if addon_short_name . split ( '<STR_LIT:->' ) [ - <NUM_LIT:1> ] == '<STR_LIT>' else '<STR_LIT>' <EOL> create_app_context ( ) <EOL> job = ArchiveJob . load ( job_pk ) <EOL> src , dst , user = job . info ( ) <EOL> src_addon = src . get_addon ( addon_name ) <EOL> try : <EOL> file_tree = src_addon . _get_file_tree ( user = user , version = version ) <EOL> except HTTPError as e : <EOL> dst . archive_job . update_target ( <EOL> addon_short_name , <EOL> ARCHIVER_NETWORK_ERROR , <EOL> errors = [ e . data [ '<STR_LIT:error>' ] ] , <EOL> ) <EOL> raise <EOL> result = AggregateStatResult ( <EOL> src_addon . _id , <EOL> addon_short_name , <EOL> targets = [ utils . aggregate_file_tree_metadata ( addon_short_name , file_tree , user ) ] , <EOL> ) <EOL> return result <EOL> @ celery_app . task ( base = ArchiverTask , name = "<STR_LIT>" ) <EOL> @ logged ( '<STR_LIT>' ) <EOL> def make_copy_request ( job_pk , url , data ) : <EOL> """<STR_LIT>""" <EOL> create_app_context ( ) <EOL> job = ArchiveJob . load ( job_pk ) <EOL> src , dst , user = job . info ( ) <EOL> provider = data [ '<STR_LIT:source>' ] [ '<STR_LIT>' ] <EOL> logger . info ( "<STR_LIT>" . format ( provider , dst . _id ) ) <EOL> res = requests . post ( url , data = json . dumps ( data ) ) <EOL> if res . status_code not in ( http . OK , http . CREATED , http . ACCEPTED ) : <EOL> raise HTTPError ( res . status_code ) <EOL> def make_waterbutler_payload ( src , dst , addon_short_name , rename , cookie , revision = None ) : <EOL> ret = { <EOL> '<STR_LIT:source>' : { <EOL> '<STR_LIT>' : cookie , <EOL> '<STR_LIT>' : src . _id , <EOL> '<STR_LIT>' : addon_short_name , <EOL> '<STR_LIT:path>' : '<STR_LIT:/>' , <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : cookie , <EOL> '<STR_LIT>' : dst . _id , <EOL> '<STR_LIT>' : settings . ARCHIVE_PROVIDER , <EOL> '<STR_LIT:path>' : '<STR_LIT:/>' , <EOL> } , <EOL> '<STR_LIT>' : rename . replace ( '<STR_LIT:/>' , '<STR_LIT:->' ) <EOL> } <EOL> if revision : <EOL> ret [ '<STR_LIT:source>' ] [ '<STR_LIT>' ] = revision <EOL> return ret <EOL> @ celery_app . task ( base = ArchiverTask , name = "<STR_LIT>" ) <EOL> @ logged ( '<STR_LIT>' ) <EOL> def archive_addon ( addon_short_name , job_pk , stat_result ) : <EOL> """<STR_LIT>""" <EOL> addon_name = addon_short_name <EOL> if '<STR_LIT>' in addon_short_name : <EOL> addon_name = '<STR_LIT>' <EOL> create_app_context ( ) <EOL> job = ArchiveJob . load ( job_pk ) <EOL> src , dst , user = job . info ( ) <EOL> logger . info ( "<STR_LIT>" . format ( addon_short_name , src . _id ) ) <EOL> src_provider = src . get_addon ( addon_name ) <EOL> folder_name = src_provider . archive_folder_name <EOL> cookie = user . get_or_create_cookie ( ) <EOL> copy_url = settings . WATERBUTLER_URL + '<STR_LIT>' <EOL> if addon_name == '<STR_LIT>' : <EOL> data = make_waterbutler_payload ( src , dst , addon_name , '<STR_LIT>' . format ( folder_name ) , cookie , revision = '<STR_LIT>' ) <EOL> make_copy_request . delay ( job_pk = job_pk , url = copy_url , data = data ) <EOL> data = make_waterbutler_payload ( src , dst , addon_name , '<STR_LIT>' . format ( folder_name ) , cookie , revision = '<STR_LIT>' ) <EOL> make_copy_request . delay ( job_pk = job_pk , url = copy_url , data = data ) <EOL> else : <EOL> data = make_waterbutler_payload ( src , dst , addon_name , folder_name , cookie ) <EOL> make_copy_request . delay ( job_pk = job_pk , url = copy_url , data = data ) <EOL> @ celery_app . task ( base = ArchiverTask , name = "<STR_LIT>" ) <EOL> @ logged ( '<STR_LIT>' ) <EOL> def archive_node ( stat_results , job_pk ) : <EOL> """<STR_LIT>""" <EOL> create_app_context ( ) <EOL> job = ArchiveJob . load ( job_pk ) <EOL> src , dst , user = job . info ( ) <EOL> logger . info ( "<STR_LIT>" . format ( src . _id ) ) <EOL> if not isinstance ( stat_results , list ) : <EOL> stat_results = [ stat_results ] <EOL> stat_result = AggregateStatResult ( <EOL> dst . _id , <EOL> dst . title , <EOL> targets = stat_results <EOL> ) <EOL> if ( NO_ARCHIVE_LIMIT not in job . initiator . system_tags ) and ( stat_result . disk_usage > settings . MAX_ARCHIVE_SIZE ) : <EOL> raise ArchiverSizeExceeded ( result = stat_result ) <EOL> else : <EOL> if not stat_result . targets : <EOL> job . status = ARCHIVER_SUCCESS <EOL> job . save ( ) <EOL> for result in stat_result . targets : <EOL> if not result . num_files : <EOL> job . update_target ( result . target_name , ARCHIVER_SUCCESS ) <EOL> else : <EOL> archive_addon . delay ( <EOL> addon_short_name = result . target_name , <EOL> job_pk = job_pk , <EOL> stat_result = result , <EOL> ) <EOL> project_signals . archive_callback . send ( dst ) <EOL> def archive ( job_pk ) : <EOL> """<STR_LIT>""" <EOL> create_app_context ( ) <EOL> job = ArchiveJob . load ( job_pk ) <EOL> src , dst , user = job . info ( ) <EOL> logger = get_task_logger ( __name__ ) <EOL> logger . info ( "<STR_LIT>" . format ( src . _id , dst . _id ) ) <EOL> return celery . chain ( <EOL> [ <EOL> celery . group ( <EOL> stat_addon . si ( <EOL> addon_short_name = target . name , <EOL> job_pk = job_pk , <EOL> ) <EOL> for target in job . target_addons <EOL> ) , <EOL> archive_node . s ( <EOL> job_pk = job_pk <EOL> ) <EOL> ] <EOL> ) <EOL> @ celery_app . task ( base = ArchiverTask , name = "<STR_LIT>" ) <EOL> @ logged ( '<STR_LIT>' ) <EOL> def archive_success ( dst_pk , job_pk ) : <EOL> """<STR_LIT>""" <EOL> create_app_context ( ) <EOL> dst = Node . load ( dst_pk ) <EOL> for schema in dst . registered_schema : <EOL> if schema . has_files : <EOL> utils . migrate_file_metadata ( dst , schema ) <EOL> job = ArchiveJob . load ( job_pk ) <EOL> if not job . sent : <EOL> job . sent = True <EOL> job . save ( ) <EOL> dst . sanction . ask ( dst . get_active_contributors_recursive ( unique_users = True ) ) </s>
