<s> import contextlib <EOL> import os <EOL> import time <EOL> from Crypto . PublicKey import RSA <EOL> from novaclient import exceptions as nova_exc <EOL> from cloudferry import cfglib <EOL> from cloudferry . lib . base import exception <EOL> from cloudferry . lib . utils import log <EOL> from cloudferry . lib . utils import proxy_client <EOL> from cloudferry . lib . utils import remote_runner <EOL> LOG = log . getLogger ( __name__ ) <EOL> RETRY_ON_EXCEPTION_DEFAULT_MAX_ATTEMPTS = <NUM_LIT:5> <EOL> INSTANCE_HOST_ATTRIBUTE = '<STR_LIT>' <EOL> INSTANCE_NAME_ATTRIBUTE = '<STR_LIT>' <EOL> NOVA_SERVICE = '<STR_LIT>' <EOL> SERVICE_DISABLED = '<STR_LIT>' <EOL> SERVICE_ENABLED = '<STR_LIT>' <EOL> ACTIVE = '<STR_LIT>' <EOL> SHUTOFF = '<STR_LIT>' <EOL> VERIFY_RESIZE = '<STR_LIT>' <EOL> SUSPENDED = '<STR_LIT>' <EOL> PAUSED = '<STR_LIT>' <EOL> ERROR = '<STR_LIT:error>' <EOL> SUPPORTED_FINAL_STATES = [ <EOL> ACTIVE , SUSPENDED , PAUSED , SHUTOFF , VERIFY_RESIZE , ERROR <EOL> ] <EOL> @ contextlib . contextmanager <EOL> def disable_all_nova_compute_services ( compute_api ) : <EOL> """<STR_LIT>""" <EOL> LOG . debug ( "<STR_LIT>" ) <EOL> services = compute_api . services . list ( binary = NOVA_SERVICE ) <EOL> temporary_disabled_services = [ ] <EOL> for service in services : <EOL> if service . status == SERVICE_DISABLED : <EOL> continue <EOL> temporary_disabled_services . append ( service . host ) <EOL> compute_api . services . disable ( service . host , NOVA_SERVICE ) <EOL> try : <EOL> yield <EOL> finally : <EOL> LOG . debug ( "<STR_LIT>" ) <EOL> for service_host in temporary_disabled_services : <EOL> compute_api . services . enable ( service_host , NOVA_SERVICE ) <EOL> @ contextlib . contextmanager <EOL> def enable_nova_compute_services ( compute_api , * hosts ) : <EOL> """<STR_LIT>""" <EOL> LOG . debug ( "<STR_LIT>" , '<STR_LIT:U+002CU+0020>' . join ( hosts ) ) <EOL> services = compute_api . services . list ( binary = NOVA_SERVICE ) <EOL> temporary_enabled_services = [ ] <EOL> for service in services : <EOL> if service . host not in hosts : <EOL> continue <EOL> if service . status == SERVICE_ENABLED : <EOL> continue <EOL> temporary_enabled_services . append ( service . host ) <EOL> compute_api . services . enable ( service . host , NOVA_SERVICE ) <EOL> try : <EOL> yield <EOL> finally : <EOL> LOG . debug ( "<STR_LIT>" , <EOL> '<STR_LIT:U+002CU+0020>' . join ( temporary_enabled_services ) ) <EOL> for service_host in temporary_enabled_services : <EOL> compute_api . services . disable ( service_host , NOVA_SERVICE ) <EOL> def is_vm_deleted ( client , instance_id ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> with proxy_client . expect_exception ( nova_exc . NotFound ) : <EOL> client . servers . get ( instance_id ) <EOL> return False <EOL> except nova_exc . NotFound : <EOL> return True <EOL> def is_vm_status_in ( client , instance_id , statuses ) : <EOL> """<STR_LIT>""" <EOL> statuses = [ s . lower ( ) for s in statuses ] <EOL> try : <EOL> with proxy_client . expect_exception ( nova_exc . NotFound ) : <EOL> instance = client . servers . get ( instance_id ) <EOL> status = instance . status . lower ( ) <EOL> if status == ERROR : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> return status in statuses <EOL> except nova_exc . NotFound : <EOL> return False <EOL> def wait_for_condition ( condition_fn , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> timeout = kwargs . pop ( '<STR_LIT>' , <NUM_LIT> ) <EOL> delay = <NUM_LIT:1> <EOL> while delay <= timeout : <EOL> if condition_fn ( * args , ** kwargs ) : <EOL> break <EOL> time . sleep ( delay ) <EOL> delay *= <NUM_LIT:2> <EOL> else : <EOL> raise exception . TimeoutException ( None , None , "<STR_LIT>" ) <EOL> def retry_on_exception ( func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> max_attempts = kwargs . pop ( '<STR_LIT>' , <EOL> RETRY_ON_EXCEPTION_DEFAULT_MAX_ATTEMPTS ) <EOL> for attempt in range ( <NUM_LIT:1> , max_attempts + <NUM_LIT:1> ) : <EOL> try : <EOL> return func ( * args , ** kwargs ) <EOL> except Exception as ex : <EOL> LOG . warning ( "<STR_LIT>" , <EOL> attempt , func . __name__ , exc_info = True ) <EOL> if attempt == max_attempts : <EOL> LOG . error ( "<STR_LIT>" <EOL> "<STR_LIT>" , func . __name__ , attempt ) <EOL> raise ex <EOL> def suppress_exceptions ( func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return func ( * args , ** kwargs ) <EOL> except Exception : <EOL> LOG . warning ( "<STR_LIT>" , exc_info = True ) <EOL> def cold_evacuate ( config , compute_api , instance_id , dst_host ) : <EOL> """<STR_LIT>""" <EOL> LOG . debug ( "<STR_LIT>" , instance_id , dst_host ) <EOL> state_change_timeout = cfglib . CONF . evacuation . state_change_timeout <EOL> migration_timeout = cfglib . CONF . evacuation . migration_timeout <EOL> if not change_to_pre_migration_state ( compute_api , instance_id ) : <EOL> instance = compute_api . servers . get ( instance_id ) <EOL> LOG . warning ( '<STR_LIT>' , instance . status ) <EOL> return <EOL> instance = compute_api . servers . get ( instance_id ) <EOL> src_host = getattr ( instance , INSTANCE_HOST_ATTRIBUTE ) <EOL> if src_host == dst_host : <EOL> LOG . warning ( "<STR_LIT>" ) <EOL> return <EOL> original_status = instance . status . lower ( ) <EOL> if original_status != SHUTOFF : <EOL> compute_api . servers . stop ( instance ) <EOL> wait_for_condition ( is_vm_status_in , compute_api , instance , <EOL> [ SHUTOFF ] , timeout = state_change_timeout ) <EOL> fix_post_cobalt_ephemeral_disk ( config , instance ) <EOL> with install_ssh_keys ( config , src_host , dst_host ) : <EOL> with disable_all_nova_compute_services ( compute_api ) : <EOL> with enable_nova_compute_services ( compute_api , dst_host , src_host ) : <EOL> compute_api . servers . migrate ( instance ) <EOL> wait_for_condition ( is_vm_status_in , compute_api , instance , <EOL> [ VERIFY_RESIZE ] , <EOL> timeout = migration_timeout ) <EOL> compute_api . servers . confirm_resize ( instance ) <EOL> wait_for_condition ( is_vm_status_in , compute_api , instance , <EOL> [ ACTIVE ] , timeout = state_change_timeout ) <EOL> if original_status == SHUTOFF . lower ( ) : <EOL> LOG . debug ( "<STR_LIT>" , instance_id ) <EOL> compute_api . servers . stop ( instance_id ) <EOL> @ contextlib . contextmanager <EOL> def install_ssh_keys ( config , * hosts ) : <EOL> """<STR_LIT>""" <EOL> ssh_user = config . cloud . ssh_user <EOL> ssh_password = config . cloud . ssh_sudo_password <EOL> home_path = cfglib . CONF . evacuation . nova_home_path <EOL> nova_user = cfglib . CONF . evacuation . nova_user <EOL> ssh_config = '<STR_LIT>' . join ( [ '<STR_LIT>' , <EOL> '<STR_LIT>' ] ) <EOL> ssh_path = '<STR_LIT:/>' . join ( [ home_path , '<STR_LIT>' ] ) <EOL> ssh_backup_base = '<STR_LIT:/>' . join ( [ home_path , '<STR_LIT>' ] ) <EOL> key = RSA . generate ( <NUM_LIT> , os . urandom ) <EOL> public_key = key . exportKey ( '<STR_LIT>' ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> private_key = key . exportKey ( '<STR_LIT>' ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> ssh_backups = { } <EOL> for host in hosts : <EOL> runner = remote_runner . RemoteRunner ( host , ssh_user , <EOL> password = ssh_password , <EOL> sudo = True ) <EOL> ssh_backup_path = '<STR_LIT:/>' . join ( [ ssh_backup_base , <EOL> os . urandom ( <NUM_LIT:8> ) . encode ( '<STR_LIT>' ) ] ) <EOL> try : <EOL> runner . run ( '<STR_LIT>' , path = ssh_path ) <EOL> runner . run ( '<STR_LIT>' , backup_base = ssh_backup_base ) <EOL> runner . run ( '<STR_LIT>' , path = ssh_path , <EOL> backup_path = ssh_backup_path ) <EOL> ssh_backups [ host ] = ssh_backup_path <EOL> except remote_runner . RemoteExecutionError : <EOL> LOG . debug ( "<STR_LIT>" ) <EOL> runner . run ( '<STR_LIT>' , path = ssh_path ) <EOL> runner . run ( '<STR_LIT>' , key = public_key , <EOL> path = '<STR_LIT:/>' . join ( [ ssh_path , '<STR_LIT>' ] ) ) <EOL> runner . run ( '<STR_LIT>' , key = private_key , <EOL> path = '<STR_LIT:/>' . join ( [ ssh_path , '<STR_LIT>' ] ) ) <EOL> runner . run ( '<STR_LIT>' , config = ssh_config , <EOL> path = '<STR_LIT:/>' . join ( [ ssh_path , '<STR_LIT>' ] ) ) <EOL> runner . run ( '<STR_LIT>' , path = '<STR_LIT:/>' . join ( [ ssh_path , '<STR_LIT>' ] ) ) <EOL> runner . run ( '<STR_LIT>' , <EOL> user = nova_user , path = ssh_path ) <EOL> try : <EOL> yield <EOL> finally : <EOL> for host in hosts : <EOL> runner = remote_runner . RemoteRunner ( host , ssh_user , <EOL> password = ssh_password , <EOL> sudo = True ) <EOL> runner . run ( '<STR_LIT>' , path = ssh_path ) <EOL> ssh_backup_path = ssh_backups . get ( host ) <EOL> if ssh_backup_path is not None : <EOL> runner . run_ignoring_errors ( <EOL> '<STR_LIT>' , <EOL> backup_path = ssh_backup_path , path = ssh_path ) <EOL> def change_to_pre_migration_state ( compute_api , instance_id ) : <EOL> """<STR_LIT>""" <EOL> state_change_timeout = cfglib . CONF . evacuation . state_change_timeout <EOL> for _ in range ( <NUM_LIT:2> ) : <EOL> instance = compute_api . servers . get ( instance_id ) <EOL> state = instance . status . lower ( ) <EOL> if state not in SUPPORTED_FINAL_STATES : <EOL> try : <EOL> LOG . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> instance_id , '<STR_LIT:U+002CU+0020>' . join ( SUPPORTED_FINAL_STATES ) ) <EOL> wait_for_condition ( is_vm_status_in , compute_api , instance_id , <EOL> SUPPORTED_FINAL_STATES , <EOL> timeout = state_change_timeout ) <EOL> except exception . TimeoutException : <EOL> return False <EOL> if state in ( ACTIVE , SHUTOFF ) : <EOL> return True <EOL> if state == ERROR : <EOL> return False <EOL> if state == VERIFY_RESIZE : <EOL> compute_api . servers . confirm_resize ( instance_id ) <EOL> if state == SUSPENDED : <EOL> compute_api . servers . resume ( instance_id ) <EOL> if state == PAUSED : <EOL> compute_api . servers . unpause ( instance_id ) <EOL> return False <EOL> def fix_post_cobalt_ephemeral_disk ( config , instance ) : <EOL> """<STR_LIT>""" <EOL> host = getattr ( instance , INSTANCE_HOST_ATTRIBUTE ) <EOL> instance_name = getattr ( instance , INSTANCE_NAME_ATTRIBUTE ) <EOL> ssh_user = config . cloud . ssh_user <EOL> ssh_password = config . cloud . ssh_sudo_password <EOL> runner = remote_runner . RemoteRunner ( host , ssh_user , <EOL> password = ssh_password , <EOL> sudo = True ) <EOL> blkinfo = runner . run ( '<STR_LIT>' , name = instance_name ) <EOL> for line in blkinfo . splitlines ( ) : <EOL> if instance . id not in line : <EOL> continue <EOL> tokens = line . split ( ) <EOL> if len ( tokens ) < <NUM_LIT:2> : <EOL> continue <EOL> _ , path = tokens [ : <NUM_LIT:2> ] <EOL> if instance . id not in path : <EOL> continue <EOL> cobalt_base_path = path + '<STR_LIT>' <EOL> info = runner . run ( '<STR_LIT>' , input = path ) <EOL> if cobalt_base_path not in info : <EOL> continue <EOL> merge_path = path + '<STR_LIT>' <EOL> runner . run ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> input = path , output = merge_path ) </s>
