<s> import logging <EOL> import time <EOL> from copy import copy <EOL> from datetime import datetime <EOL> from boto import ec2 , rds <EOL> from neckbeard . cloud_resource import InfrastructureNode <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> WAIT_TIME = <NUM_LIT:10> <EOL> MAKE_OPERATIONAL_TIMEOUT = <NUM_LIT:4> * <NUM_LIT> <EOL> class PstatRdsId ( object ) : <EOL> def __init__ ( self , pstat_instance , pstat_version , counter ) : <EOL> self . pstat_instance = pstat_instance <EOL> self . pstat_version = pstat_version <EOL> self . counter = int ( counter ) <EOL> def __str__ ( self ) : <EOL> rds_tpl = u'<STR_LIT>' <EOL> rds_args = ( <EOL> self . pstat_instance , <EOL> self . pstat_version . replace ( '<STR_LIT:.>' , '<STR_LIT:->' ) , <EOL> self . counter , <EOL> ) <EOL> return rds_tpl % rds_args <EOL> @ staticmethod <EOL> def from_string ( id_str ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> pstat_instance , tail = id_str . split ( '<STR_LIT>' ) <EOL> pstat_version , counter = tail . split ( '<STR_LIT>' ) <EOL> counter = int ( counter ) <EOL> return PstatRdsId ( pstat_instance , pstat_version , counter ) <EOL> except ValueError : <EOL> pass <EOL> return None <EOL> class WaitTimedOut ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class MissingAWSCredentials ( Exception ) : <EOL> pass <EOL> class NonUniformAWSCredentials ( Exception ) : <EOL> pass <EOL> class Deployment ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , deployment_name , ec2_nodes , rds_nodes , elb_nodes ) : <EOL> """<STR_LIT>""" <EOL> self . deployment_name = deployment_name <EOL> self . deployment_confs = { } <EOL> self . deployment_confs [ '<STR_LIT>' ] = ec2_nodes <EOL> self . deployment_confs [ '<STR_LIT>' ] = rds_nodes <EOL> self . deployment_confs [ '<STR_LIT>' ] = elb_nodes <EOL> aws_credentials = self . _get_valid_aws_credentials ( <EOL> self . deployment_confs , <EOL> ) <EOL> self . _pending_gen_id = None <EOL> self . _active_gen_id = None <EOL> self . ec2conn = ec2 . EC2Connection ( <EOL> aws_credentials [ '<STR_LIT>' ] , <EOL> aws_credentials [ '<STR_LIT>' ] , <EOL> ) <EOL> self . rdsconn = rds . RDSConnection ( <EOL> aws_credentials [ '<STR_LIT>' ] , <EOL> aws_credentials [ '<STR_LIT>' ] , <EOL> ) <EOL> def _get_valid_aws_credentials ( self , deployment_confs ) : <EOL> aws_credentials = { <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : None , <EOL> } <EOL> for aws_type , configs_of_type in deployment_confs . items ( ) : <EOL> for name , resource_config in configs_of_type . items ( ) : <EOL> aws_config = resource_config . get ( '<STR_LIT>' ) <EOL> if aws_config is None : <EOL> raise MissingAWSCredentials ( <EOL> "<STR_LIT>" % ( <EOL> aws_type , <EOL> name , <EOL> ) , <EOL> ) <EOL> for key in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> value = aws_config . get ( key ) <EOL> if value is None : <EOL> raise MissingAWSCredentials ( <EOL> "<STR_LIT>" % ( <EOL> aws_type , <EOL> name , <EOL> key , <EOL> ) , <EOL> ) <EOL> other_resource_value = aws_credentials . get ( key ) <EOL> if other_resource_value is not None : <EOL> if value != other_resource_value : <EOL> raise NonUniformAWSCredentials ( <EOL> ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> ) % ( <EOL> aws_type , <EOL> name , <EOL> key , <EOL> ) , <EOL> ) <EOL> aws_credentials [ key ] = value <EOL> return aws_credentials <EOL> @ property <EOL> def active_gen_id ( self ) : <EOL> if self . _active_gen_id : <EOL> return self . _active_gen_id <EOL> active_nodes = InfrastructureNode . objects . filter ( <EOL> deployment_name = self . deployment_name , <EOL> is_active_generation = <NUM_LIT:1> ) <EOL> if len ( active_nodes ) == <NUM_LIT:0> : <EOL> return None <EOL> first_active_node = active_nodes [ <NUM_LIT:0> ] <EOL> gen_id = first_active_node . generation_id <EOL> for active_node in active_nodes : <EOL> if active_node . generation_id != gen_id : <EOL> err_str = ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> ) <EOL> context = ( <EOL> first_active_node . aws_id , <EOL> first_active_node . generation_id , <EOL> active_node . aws_id , <EOL> active_node . generation_id ) <EOL> raise Exception ( err_str % context ) <EOL> self . _active_gen_id = gen_id <EOL> return self . _active_gen_id <EOL> @ property <EOL> def pending_gen_id ( self ) : <EOL> if self . _pending_gen_id : <EOL> return self . _pending_gen_id <EOL> if self . active_gen_id : <EOL> self . _pending_gen_id = self . active_gen_id + <NUM_LIT:1> <EOL> return self . _pending_gen_id <EOL> self . _pending_gen_id = <NUM_LIT:1> <EOL> return self . _pending_gen_id <EOL> def get_blank_node ( self , aws_type ) : <EOL> node = InfrastructureNode ( ) <EOL> node . set_aws_conns ( self . ec2conn , self . rdsconn ) <EOL> node . aws_type = aws_type <EOL> return node <EOL> def get_active_node ( self , aws_type , node_name ) : <EOL> if self . active_gen_id : <EOL> return self . get_node ( <EOL> aws_type , node_name , self . active_gen_id , is_running = <NUM_LIT:1> ) <EOL> return None <EOL> def get_pending_node ( self , aws_type , node_name ) : <EOL> return self . get_node ( <EOL> aws_type , node_name , self . pending_gen_id , is_running = <NUM_LIT:1> ) <EOL> def get_all_active_nodes ( self , is_running = None ) : <EOL> if self . active_gen_id : <EOL> return self . get_all_nodes ( <EOL> self . active_gen_id , is_running = is_running ) <EOL> return [ ] <EOL> def get_all_pending_nodes ( self , is_running = None ) : <EOL> return self . get_all_nodes ( self . pending_gen_id , is_running = is_running ) <EOL> def get_all_old_nodes ( self , is_running = None ) : <EOL> nodes = [ ] <EOL> all_nodes = self . get_all_nodes ( ) <EOL> new_generations = [ self . active_gen_id , self . pending_gen_id ] <EOL> for node in all_nodes : <EOL> if node . generation_id not in new_generations : <EOL> nodes . append ( node ) <EOL> nodes . sort ( key = lambda n : n . generation_id ) <EOL> return nodes <EOL> def get_all_nodes ( self , generation_id = None , is_running = None ) : <EOL> matching_nodes = InfrastructureNode . objects . filter ( <EOL> deployment_name = self . deployment_name , <EOL> ) <EOL> if generation_id : <EOL> matching_nodes = matching_nodes . filter ( generation_id = generation_id ) <EOL> if is_running is not None : <EOL> matching_nodes = matching_nodes . filter ( is_running = is_running ) <EOL> configured_nodes = [ ] <EOL> for node in matching_nodes : <EOL> node . set_aws_conns ( self . ec2conn , self . rdsconn ) <EOL> deploy_conf = self . deployment_confs [ node . aws_type ] . get ( <EOL> node . name , None ) <EOL> if deploy_conf is None : <EOL> if node . is_running : <EOL> logger . warning ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> self . deployment_name , <EOL> node . aws_type , <EOL> node . name , <EOL> ) <EOL> logger . info ( <EOL> "<STR_LIT>" , <EOL> self . deployment_confs [ node . aws_type ] . keys ( ) , <EOL> ) <EOL> else : <EOL> node . set_deployment_info ( <EOL> self . deployment_confs [ node . aws_type ] [ node . name ] ) <EOL> configured_nodes . append ( node ) <EOL> return configured_nodes <EOL> def get_node ( self , aws_type , node_name , generation_id , is_running = <NUM_LIT:1> ) : <EOL> matching_nodes = InfrastructureNode . objects . filter ( <EOL> deployment_name = self . deployment_name , <EOL> generation_id = generation_id , <EOL> aws_type = aws_type , <EOL> name = node_name , <EOL> is_running = is_running , <EOL> ) <EOL> if len ( matching_nodes ) > <NUM_LIT:1> : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> elif len ( matching_nodes ) == <NUM_LIT:1> : <EOL> node = matching_nodes [ <NUM_LIT:0> ] <EOL> node . set_aws_conns ( self . ec2conn , self . rdsconn ) <EOL> node . set_deployment_info ( <EOL> self . deployment_confs [ node . aws_type ] [ node . name ] ) <EOL> return node <EOL> else : <EOL> return None <EOL> def set_active_node ( self , aws_type , node_name , boto_object ) : <EOL> if self . active_gen_id : <EOL> gen_id = self . active_gen_id <EOL> else : <EOL> gen_id = self . pending_gen_id <EOL> return self . set_node ( <EOL> aws_type , <EOL> node_name , <EOL> gen_id , <EOL> boto_object , <EOL> is_active = True ) <EOL> def set_pending_node ( self , aws_type , node_name , boto_object ) : <EOL> return self . set_node ( <EOL> aws_type , <EOL> node_name , <EOL> self . pending_gen_id , <EOL> boto_object , <EOL> is_active = False ) <EOL> def set_node ( <EOL> self , aws_type , node_name , generation_id , boto_object , is_active , <EOL> ) : <EOL> aws_id = boto_object . id <EOL> matching_nodes = InfrastructureNode . objects . filter ( <EOL> aws_type = aws_type , <EOL> aws_id = aws_id , <EOL> deployment_name = self . deployment_name , <EOL> ) <EOL> if len ( matching_nodes ) == <NUM_LIT:1> : <EOL> node = matching_nodes [ <NUM_LIT:0> ] <EOL> else : <EOL> node = self . get_blank_node ( aws_type ) <EOL> node . generation_id = generation_id <EOL> node . deployment_name = self . deployment_name <EOL> node . aws_type = aws_type <EOL> node . aws_id = aws_id <EOL> node . name = node_name <EOL> node . creation_date = datetime . now ( ) <EOL> node . is_running = <NUM_LIT:1> <EOL> node . is_active_generation = is_active <EOL> node . save ( ) <EOL> def get_new_rds_label ( self , node_name , version , is_active = False ) : <EOL> counter = self . pending_gen_id <EOL> if is_active : <EOL> counter = self . active_gen_id <EOL> rds_label = PstatRdsId ( <EOL> '<STR_LIT>' + self . deployment_name , version , counter ) <EOL> return rds_label <EOL> def verify_running_state ( self , nodes ) : <EOL> for node in nodes : <EOL> node . verify_running_state ( ) <EOL> def verify_active_deployment_state ( self ) : <EOL> """<STR_LIT>""" <EOL> nodes = self . get_all_active_nodes ( ) <EOL> self . verify_running_state ( nodes ) <EOL> def verify_pending_deployment_state ( self ) : <EOL> nodes = self . get_all_pending_nodes ( ) <EOL> self . verify_running_state ( nodes ) <EOL> def verify_old_deployment_state ( self ) : <EOL> nodes = self . get_all_old_nodes ( ) <EOL> self . verify_running_state ( nodes ) <EOL> def verify_deployment_state ( self , verify_old = True ) : <EOL> """<STR_LIT>""" <EOL> self . verify_pending_deployment_state ( ) <EOL> self . verify_active_deployment_state ( ) <EOL> if verify_old : <EOL> self . verify_old_deployment_state ( ) <EOL> def get_inoperational_active_nodes ( self ) : <EOL> """<STR_LIT>""" <EOL> inoperational = [ ] <EOL> for aws_type , confs in self . deployment_confs . items ( ) : <EOL> for node_name , node_confs in confs . items ( ) : <EOL> node = self . get_active_node ( aws_type , node_name ) <EOL> if not node : <EOL> mock_node = self . get_blank_node ( aws_type ) <EOL> mock_node . name = node_name <EOL> inoperational . append ( mock_node ) <EOL> logger . info ( "<STR_LIT>" % ( aws_type , node_name ) ) <EOL> continue <EOL> if not node . is_operational : <EOL> inoperational . append ( node ) <EOL> return inoperational <EOL> def get_unhealthy_active_nodes ( self ) : <EOL> """<STR_LIT>""" <EOL> nodes = self . get_all_active_nodes ( ) <EOL> unhealthy = [ <EOL> node for node in nodes <EOL> if node . is_running and not node . is_healthy <EOL> ] <EOL> for aws_type , confs in self . deployment_confs . items ( ) : <EOL> for node_name , node_confs in confs . items ( ) : <EOL> node = self . get_active_node ( aws_type , node_name ) <EOL> if not node : <EOL> mock_node = self . get_blank_node ( aws_type ) <EOL> mock_node . name = node_name <EOL> unhealthy . append ( mock_node ) <EOL> logger . info ( "<STR_LIT>" % ( aws_type , node_name ) ) <EOL> continue <EOL> if not node . is_healthy : <EOL> unhealthy . append ( node ) <EOL> logger . info ( "<STR_LIT>" % node ) <EOL> return unhealthy <EOL> def get_unhealthy_pending_nodes ( self ) : <EOL> """<STR_LIT>""" <EOL> nodes = self . get_all_pending_nodes ( ) <EOL> unhealthy = [ <EOL> node for node in nodes <EOL> if node . is_running and not node . is_healthy <EOL> ] <EOL> for aws_type , confs in self . deployment_confs . items ( ) : <EOL> for node_name , node_confs in confs . items ( ) : <EOL> node = self . get_pending_node ( aws_type , node_name ) <EOL> if not node : <EOL> mock_node = self . get_blank_node ( aws_type ) <EOL> mock_node . name = node_name <EOL> unhealthy . append ( mock_node ) <EOL> logger . info ( "<STR_LIT>" % ( aws_type , node_name ) ) <EOL> continue <EOL> if not node . is_healthy : <EOL> unhealthy . append ( node ) <EOL> logger . info ( "<STR_LIT>" % node ) <EOL> return unhealthy <EOL> def active_is_healthy ( self ) : <EOL> """<STR_LIT>""" <EOL> unhealthy_nodes = self . get_unhealthy_active_nodes ( ) <EOL> if unhealthy_nodes : <EOL> return False <EOL> return True <EOL> def active_is_fully_operational ( self ) : <EOL> inoperational_nodes = self . get_inoperational_active_nodes ( ) <EOL> if inoperational_nodes : <EOL> return False <EOL> return True <EOL> def pending_is_healthy ( self ) : <EOL> """<STR_LIT>""" <EOL> unhealthy_nodes = self . get_unhealthy_pending_nodes ( ) <EOL> if unhealthy_nodes : <EOL> return False <EOL> return True <EOL> def repair_active_generation ( <EOL> self , force_operational = False , wait_until_operational = True , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> if self . active_is_fully_operational ( ) : <EOL> logger . info ( "<STR_LIT>" ) <EOL> return [ ] <EOL> nodes = self . get_inoperational_active_nodes ( ) <EOL> made_operational = [ ] <EOL> for node in nodes : <EOL> if not node . aws_id : <EOL> logger . warning ( <EOL> "<STR_LIT>" , <EOL> node . aws_type , <EOL> node . name , <EOL> ) <EOL> continue <EOL> if not node . is_operational : <EOL> if node . is_healthy or force_operational : <EOL> logger . info ( "<STR_LIT>" % node ) <EOL> node . make_operational ( force_operational = force_operational ) <EOL> made_operational . append ( node ) <EOL> else : <EOL> logger . warning ( "<STR_LIT>" % node . boto_instance ) <EOL> else : <EOL> logger . debug ( "<STR_LIT>" % node ) <EOL> fixed_nodes = copy ( made_operational ) <EOL> if len ( fixed_nodes ) == <NUM_LIT:0> : <EOL> logger . info ( "<STR_LIT>" ) <EOL> return [ ] <EOL> if not wait_until_operational : <EOL> return fixed_nodes <EOL> logger . info ( "<STR_LIT>" ) <EOL> time_waited = <NUM_LIT:0> <EOL> while made_operational : <EOL> if time_waited > MAKE_OPERATIONAL_TIMEOUT : <EOL> raise WaitTimedOut ( <EOL> "<STR_LIT>" % made_operational ) <EOL> _made_operational = made_operational [ : ] <EOL> for node in _made_operational : <EOL> if node . is_operational : <EOL> made_operational . remove ( node ) <EOL> if made_operational : <EOL> logger . info ( <EOL> "<STR_LIT>" , <EOL> len ( made_operational ) , <EOL> WAIT_TIME , <EOL> ) <EOL> time . sleep ( WAIT_TIME ) <EOL> time_waited += WAIT_TIME <EOL> return fixed_nodes <EOL> def uses_rds ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . deployment_confs [ '<STR_LIT>' ] . keys ( ) ) == <NUM_LIT:0> : <EOL> return False <EOL> return True <EOL> def has_required_redundancy ( self , node ) : <EOL> """<STR_LIT>""" <EOL> all_gen_nodes = self . get_all_nodes ( node . generation_id , is_running = True ) <EOL> same_aws_nodes = [ <EOL> n <EOL> for n in all_gen_nodes <EOL> if n . aws_type == node . aws_type <EOL> ] <EOL> other_operational_nodes = [ ] <EOL> for other_node in same_aws_nodes : <EOL> if other_node . name != node . name : <EOL> if other_node . is_operational : <EOL> other_operational_nodes . append ( other_node ) <EOL> return bool ( other_operational_nodes ) <EOL> def increment_generation ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . pending_is_healthy ( ) : <EOL> raise Exception ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> ) <EOL> active_nodes = self . get_all_active_nodes ( ) <EOL> pending_nodes = self . get_all_pending_nodes ( ) <EOL> for node in active_nodes : <EOL> node . is_active_generation = <NUM_LIT:0> <EOL> node . save ( ) <EOL> for node in pending_nodes : <EOL> node . is_active_generation = <NUM_LIT:1> <EOL> node . save ( ) <EOL> self . _active_gen_id += <NUM_LIT:1> <EOL> self . _pending_gen_id += <NUM_LIT:1> <EOL> logger . info ( "<STR_LIT>" ) <EOL> logger . info ( "<STR_LIT>" ) <EOL> self . repair_active_generation ( ) </s>
