<s> import logging <EOL> import sys <EOL> from dropbox . client import DropboxClient <EOL> from nose . tools import * <EOL> from modularodm import Q <EOL> from framework . mongo import database <EOL> from framework . transactions . context import TokuTransaction <EOL> from website . app import init_app <EOL> from website . models import User , Node <EOL> from website . oauth . models import ExternalAccount <EOL> from scripts import utils as script_utils <EOL> logger = logging . getLogger ( __name__ ) <EOL> PROVIDER = '<STR_LIT>' <EOL> PROVIDER_NAME = '<STR_LIT>' <EOL> def verify_user_settings_document ( document ) : <EOL> try : <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> if document [ '<STR_LIT>' ] : <EOL> assert_is_not_none ( document [ '<STR_LIT>' ] ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document [ '<STR_LIT>' ] ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_is_not_none ( document [ '<STR_LIT>' ] ) <EOL> except AssertionError : <EOL> return salvage_broken_user_settings_document ( document ) <EOL> else : <EOL> return True <EOL> def verify_node_settings_document ( document ) : <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> assert_is_not_none ( document [ '<STR_LIT>' ] ) <EOL> assert_in ( '<STR_LIT>' , document ) <EOL> def salvage_broken_user_settings_document ( document ) : <EOL> if not document [ '<STR_LIT>' ] or not document [ '<STR_LIT>' ] : <EOL> return False <EOL> if not document [ '<STR_LIT>' ] or not User . load ( document [ '<STR_LIT>' ] ) . is_active : <EOL> return False <EOL> if document [ '<STR_LIT>' ] : <EOL> return False <EOL> if not document . get ( '<STR_LIT>' ) or not document [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> logger . info ( <EOL> "<STR_LIT>" . format ( document [ '<STR_LIT>' ] ) <EOL> ) <EOL> client = DropboxClient ( document [ '<STR_LIT>' ] ) <EOL> document [ '<STR_LIT>' ] = { } <EOL> try : <EOL> database [ '<STR_LIT>' ] . find_and_modify ( <EOL> { '<STR_LIT>' : document [ '<STR_LIT>' ] } , <EOL> { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : client . account_info ( ) <EOL> } <EOL> } <EOL> ) <EOL> except Exception : <EOL> return True <EOL> else : <EOL> return True <EOL> return False <EOL> def migrate_to_external_account ( user_settings_document ) : <EOL> if not user_settings_document . get ( '<STR_LIT>' ) : <EOL> return ( None , None , None ) <EOL> new = False <EOL> user = User . load ( user_settings_document [ '<STR_LIT>' ] ) <EOL> try : <EOL> external_account = ExternalAccount . find ( Q ( '<STR_LIT>' , '<STR_LIT>' , user_settings_document [ '<STR_LIT>' ] ) ) [ <NUM_LIT:0> ] <EOL> logger . info ( '<STR_LIT>' . format ( user . username , user_settings_document [ '<STR_LIT>' ] ) ) <EOL> except IndexError : <EOL> new = True <EOL> external_account = ExternalAccount ( <EOL> provider = PROVIDER , <EOL> provider_name = PROVIDER_NAME , <EOL> provider_id = user_settings_document [ '<STR_LIT>' ] , <EOL> oauth_key = user_settings_document [ '<STR_LIT>' ] , <EOL> display_name = user_settings_document [ '<STR_LIT>' ] . get ( '<STR_LIT>' , None ) if user_settings_document . get ( '<STR_LIT>' , None ) else None , <EOL> ) <EOL> external_account . save ( ) <EOL> user . external_accounts . append ( external_account ) <EOL> user . save ( ) <EOL> return external_account , user , new <EOL> def make_new_user_settings ( user ) : <EOL> database [ '<STR_LIT:user>' ] . find_and_modify ( <EOL> { '<STR_LIT>' : user . _id } , <EOL> { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> } <EOL> ) <EOL> user . reload ( ) <EOL> return user . get_or_add_addon ( '<STR_LIT>' , override = True ) <EOL> def make_new_node_settings ( node , node_settings_document , external_account = None , user_settings_instance = None ) : <EOL> database [ '<STR_LIT>' ] . find_and_modify ( <EOL> { '<STR_LIT>' : node . _id } , <EOL> { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> } <EOL> ) <EOL> node . reload ( ) <EOL> node_settings_instance = node . get_or_add_addon ( '<STR_LIT>' , auth = None , override = True , log = False ) <EOL> node_settings_instance . folder = node_settings_document [ '<STR_LIT>' ] <EOL> node_settings_instance . save ( ) <EOL> if external_account and user_settings_instance : <EOL> node_settings_instance . set_auth ( <EOL> external_account , <EOL> user_settings_instance . owner , <EOL> log = False <EOL> ) <EOL> return node_settings_instance <EOL> def remove_old_documents ( old_user_settings , old_user_settings_count , old_node_settings , old_node_settings_count , dry_run ) : <EOL> logger . info ( '<STR_LIT>' . format ( <EOL> old_user_settings_count <EOL> ) ) <EOL> if not dry_run : <EOL> database [ '<STR_LIT>' ] . remove ( { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : map ( lambda s : s [ '<STR_LIT>' ] , old_user_settings ) <EOL> } <EOL> } ) <EOL> logger . info ( '<STR_LIT>' . format ( <EOL> old_node_settings_count <EOL> ) ) <EOL> if not dry_run : <EOL> database [ '<STR_LIT>' ] . remove ( { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : map ( lambda s : s [ '<STR_LIT>' ] , old_node_settings ) <EOL> } <EOL> } ) <EOL> def migrate ( dry_run = True , remove_old = True ) : <EOL> rm_msg = '<STR_LIT>' if remove_old else '<STR_LIT>' <EOL> user_settings_list = list ( database [ '<STR_LIT>' ] . find ( ) ) <EOL> user_settings_collection = database [ '<STR_LIT>' ] <EOL> old_user_settings = list ( user_settings_collection . find ( ) ) <EOL> old_user_settings_count = user_settings_collection . count ( ) <EOL> node_settings_collection = database [ '<STR_LIT>' ] <EOL> old_node_settings = list ( node_settings_collection . find ( ) ) <EOL> old_node_settings_count = node_settings_collection . count ( ) <EOL> external_accounts_created = <NUM_LIT:0> <EOL> migrated_user_settings = <NUM_LIT:0> <EOL> migrated_node_settings = <NUM_LIT:0> <EOL> for user_settings_document in user_settings_list : <EOL> if not verify_user_settings_document ( user_settings_document ) : <EOL> logger . info ( <EOL> "<STR_LIT>" . format ( user_settings_document [ '<STR_LIT>' ] , rm_msg ) <EOL> ) <EOL> continue <EOL> if user_settings_document [ '<STR_LIT>' ] : <EOL> logger . info ( <EOL> "<STR_LIT>" . format ( user_settings_document [ '<STR_LIT>' ] , rm_msg ) <EOL> ) <EOL> continue <EOL> user_settings_document = database [ '<STR_LIT>' ] . find_one ( { '<STR_LIT>' : user_settings_document [ '<STR_LIT>' ] } ) <EOL> external_account , user , new = migrate_to_external_account ( user_settings_document ) <EOL> if not external_account : <EOL> logger . info ( "<STR_LIT>" . format ( <EOL> user_settings_document [ '<STR_LIT>' ] <EOL> ) ) <EOL> else : <EOL> if new : <EOL> external_accounts_created += <NUM_LIT:1> <EOL> linked_node_settings_documents = database [ '<STR_LIT>' ] . find ( { <EOL> '<STR_LIT>' : user_settings_document [ '<STR_LIT>' ] <EOL> } ) <EOL> if not user or not user . is_active : <EOL> if linked_node_settings_documents . count ( ) and not user . is_merged : <EOL> logger . warn ( "<STR_LIT>" . format ( <EOL> user_settings_document [ '<STR_LIT>' ] , <EOL> '<STR_LIT:U+002CU+0020>' . join ( [ each [ '<STR_LIT>' ] for each in linked_node_settings_documents ] ) <EOL> ) ) <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> else : <EOL> logger . info ( "<STR_LIT>" . format ( <EOL> user_settings_document [ '<STR_LIT>' ] <EOL> ) ) <EOL> continue <EOL> else : <EOL> user_settings_instance = make_new_user_settings ( user ) <EOL> for node_settings_document in linked_node_settings_documents : <EOL> verify_node_settings_document ( node_settings_document ) <EOL> if node_settings_document [ '<STR_LIT>' ] : <EOL> logger . info ( <EOL> "<STR_LIT>" . format ( <EOL> node_settings_document [ '<STR_LIT>' ] , <EOL> rm_msg <EOL> ) <EOL> ) <EOL> continue <EOL> node = Node . load ( node_settings_document [ '<STR_LIT>' ] ) <EOL> if not node : <EOL> logger . info ( "<STR_LIT>" . format ( <EOL> node_settings_document [ '<STR_LIT>' ] <EOL> ) ) <EOL> continue <EOL> else : <EOL> make_new_node_settings ( <EOL> node , <EOL> node_settings_document , <EOL> external_account , <EOL> user_settings_instance <EOL> ) <EOL> migrated_node_settings += <NUM_LIT:1> <EOL> migrated_user_settings += <NUM_LIT:1> <EOL> logger . info ( <EOL> "<STR_LIT>" . format ( <EOL> external_accounts_created , migrated_user_settings , migrated_node_settings <EOL> ) <EOL> ) <EOL> if remove_old : <EOL> remove_old_documents ( <EOL> old_user_settings , old_user_settings_count , <EOL> old_node_settings , old_node_settings_count , <EOL> dry_run <EOL> ) <EOL> if dry_run : <EOL> raise RuntimeError ( '<STR_LIT>' ) <EOL> def main ( ) : <EOL> dry_run = False <EOL> remove_old = True <EOL> if '<STR_LIT>' in sys . argv : <EOL> remove_old = False <EOL> if '<STR_LIT>' in sys . argv : <EOL> dry_run = True <EOL> if not dry_run : <EOL> script_utils . add_file_logger ( logger , __file__ ) <EOL> init_app ( set_backends = True , routes = False ) <EOL> with TokuTransaction ( ) : <EOL> migrate ( dry_run = dry_run , remove_old = remove_old ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
