<s> """<STR_LIT>""" <EOL> import hashlib <EOL> import logging <EOL> from os import path <EOL> import re <EOL> import time <EOL> import consulate <EOL> import fleetpy <EOL> import yaml <EOL> from houston import files <EOL> from houston import utils <EOL> LOGGER = logging . getLogger ( __name__ ) <EOL> CONFIG_FILE = '<STR_LIT>' <EOL> UNIT_PATTERN = re . compile ( r'<STR_LIT>' <EOL> r'<STR_LIT>' ) <EOL> class Controller ( object ) : <EOL> def __init__ ( self , config_path , environment , command , name , group , version , <EOL> delay , max_tries , no_dependencies , no_removal , skip_consul , <EOL> remove_units ) : <EOL> self . _config_path = self . _normalize_path ( config_path ) <EOL> self . _environment = environment <EOL> self . _command = command <EOL> self . _name = name or ( '<STR_LIT>' if command != '<STR_LIT>' else '<STR_LIT>' ) <EOL> self . _group = group <EOL> self . _version = version <EOL> self . _deployed_units = [ ] <EOL> self . _delay = delay <EOL> self . _max_tries = max_tries <EOL> self . _no_dependencies = no_dependencies <EOL> self . _no_removal = no_removal <EOL> self . _skip_consul = skip_consul <EOL> self . _remove_unit_files = remove_units <EOL> self . _config = self . _load_config ( CONFIG_FILE ) <EOL> if environment not in self . _config . get ( '<STR_LIT>' , { } ) : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> kwargs = utils . parse_endpoint ( self . env_config [ '<STR_LIT>' ] ) <EOL> self . _consul = consulate . Consul ( ** kwargs ) <EOL> self . _file_deployment = None <EOL> self . _fleet = fleetpy . Client ( self . env_config . get ( '<STR_LIT>' ) ) <EOL> @ property <EOL> def env_config ( self ) : <EOL> return self . _config . get ( '<STR_LIT>' , { } ) . get ( self . _environment , { } ) <EOL> def run ( self ) : <EOL> if self . _remove_unit_files : <EOL> return self . _remove_units ( ) <EOL> name = self . _name <EOL> if self . _version : <EOL> name = '<STR_LIT>' . format ( name , self . _version ) <EOL> if self . _group : <EOL> name = '<STR_LIT>' . format ( name , self . _group ) <EOL> LOGGER . info ( '<STR_LIT>' , name ) <EOL> if self . _global : <EOL> self . _name = '<STR_LIT>' <EOL> if not self . _deploy_globals ( ) : <EOL> return False <EOL> if not self . _deploy_files ( ) : <EOL> return False <EOL> self . _shutdown_other_versions ( ) <EOL> if self . _file_deployment : <EOL> self . _file_deployment . remove_other_archive_versions ( ) <EOL> return True <EOL> self . _add_last_deployed_global ( ) <EOL> if not self . _deploy_files ( ) : <EOL> LOGGER . info ( '<STR_LIT>' ) <EOL> return False <EOL> if self . _standalone : <EOL> if not self . _deploy_shared_units ( ) : <EOL> return False <EOL> self . _shutdown_other_versions ( ) <EOL> if self . _file_deployment : <EOL> self . _file_deployment . remove_other_archive_versions ( ) <EOL> return True <EOL> return self . _deploy_service ( ) <EOL> def _add_last_deployed_global ( self ) : <EOL> """<STR_LIT>""" <EOL> service = self . _config . get ( '<STR_LIT>' , [ ] ) [ - <NUM_LIT:1> ] <EOL> version = None <EOL> if '<STR_LIT::>' in service : <EOL> service , version = service . split ( '<STR_LIT::>' ) <EOL> self . _deployed_units . append ( utils . unit_name ( service , '<STR_LIT>' , <EOL> self . _group , version ) ) <EOL> def _apply_variables ( self , value ) : <EOL> value = value . replace ( '<STR_LIT>' , self . _name ) <EOL> if self . _group : <EOL> value = value . replace ( '<STR_LIT>' , self . _group ) <EOL> variables = self . _config . get ( '<STR_LIT>' , { } ) <EOL> if self . _environment in variables : <EOL> variables = variables [ self . _environment ] <EOL> for name in variables : <EOL> key = '<STR_LIT>' . format ( name ) <EOL> if key in value : <EOL> value = value . replace ( key , variables [ name ] ) <EOL> for unit_name in self . _deployed_units : <EOL> _parent , service , _group , _ver = utils . parse_unit_name ( unit_name ) <EOL> base_service = '<STR_LIT>' . format ( service ) <EOL> if base_service in value : <EOL> LOGGER . debug ( '<STR_LIT>' , <EOL> base_service , unit_name ) <EOL> value = value . replace ( base_service , unit_name ) <EOL> return value <EOL> def _check_consul_for_service ( self ) : <EOL> """<STR_LIT>""" <EOL> version = '<STR_LIT>' . format ( self . _version or '<STR_LIT>' ) <EOL> group = '<STR_LIT>' . format ( self . _group ) if self . _group else None <EOL> unit_name = utils . unit_name ( self . _name , None , self . _group , <EOL> self . _version ) <EOL> state = self . _fleet . state ( True , unit_name ) <EOL> expected = set ( [ s . ipaddr for s in state ] ) <EOL> LOGGER . debug ( '<STR_LIT>' , <EOL> len ( expected ) ) <EOL> running = self . _consul . catalog . service ( self . _name ) <EOL> actual = set ( ) <EOL> for node in running : <EOL> if group and group not in node [ '<STR_LIT>' ] : <EOL> continue <EOL> elif version not in node [ '<STR_LIT>' ] : <EOL> continue <EOL> else : <EOL> actual . add ( node [ '<STR_LIT>' ] ) <EOL> LOGGER . debug ( '<STR_LIT>' , <EOL> len ( actual ) , self . _name , version ) <EOL> return ( expected & actual ) == expected <EOL> def _deploy_files ( self ) : <EOL> if self . _file_manifest ( ) : <EOL> if self . _global : <EOL> manifest = '<STR_LIT>' <EOL> else : <EOL> manifest = '<STR_LIT>' . format ( self . _command , self . _name ) <EOL> unit_name = utils . unit_name ( '<STR_LIT>' , self . _name , self . _group , <EOL> self . _file_manifest_hash ( ) ) <EOL> self . _file_deployment = files . FileDeployment ( unit_name , <EOL> self . env_config , <EOL> self . _config_path , <EOL> manifest , <EOL> self . _name , <EOL> self . _group , <EOL> self . _environment ) <EOL> if self . _unit_is_active ( unit_name ) : <EOL> self . _deployed_units . append ( unit_name ) <EOL> return True <EOL> if self . _file_deployment . build_archive ( ) : <EOL> LOGGER . info ( '<STR_LIT>' ) <EOL> self . _file_deployment . upload_archive ( ) <EOL> unit = self . _fleet . unit ( unit_name ) <EOL> unit . read_string ( self . _file_deployment . unit_file ( ) ) <EOL> if self . _deployed_units : <EOL> self . _maybe_add_last_unit ( unit , self . _deployed_units [ - <NUM_LIT:1> ] ) <EOL> LOGGER . info ( '<STR_LIT>' , unit_name ) <EOL> unit . submit ( ) <EOL> unit . start ( ) <EOL> if not self . _wait_for_unit_to_become_active ( unit_name ) : <EOL> LOGGER . error ( '<STR_LIT>' ) <EOL> return False <EOL> self . _deployed_units . append ( unit_name ) <EOL> return True <EOL> def _deploy_globals ( self ) : <EOL> last_unit = None <EOL> global_unit_prefix = path . join ( self . _config_path , '<STR_LIT>' , '<STR_LIT>' ) <EOL> for service in self . _config . get ( '<STR_LIT>' , [ ] ) : <EOL> version = None <EOL> if '<STR_LIT::>' in service : <EOL> service , version = service . split ( '<STR_LIT::>' ) <EOL> unit_file = path . join ( global_unit_prefix , service ) <EOL> unit_name = utils . unit_name ( service , '<STR_LIT>' , self . _group , version ) <EOL> if not self . _deploy_unit ( unit_name , unit_file , last_unit ) : <EOL> LOGGER . error ( '<STR_LIT>' , service ) <EOL> return False <EOL> last_unit = unit_name <EOL> self . _shutdown_other_versions ( ) <EOL> if self . _file_deployment : <EOL> self . _file_deployment . remove_other_archive_versions ( ) <EOL> return True <EOL> def _deploy_service ( self ) : <EOL> if not self . _deploy_shared_units ( ) : <EOL> LOGGER . info ( '<STR_LIT>' ) <EOL> return False <EOL> unit_file = path . join ( self . _config_path , '<STR_LIT>' , '<STR_LIT>' , <EOL> self . _name ) <EOL> unit_name = utils . unit_name ( self . _name , None , self . _group , <EOL> self . _version ) <EOL> if not self . _deploy_unit ( unit_name , unit_file ) : <EOL> LOGGER . info ( '<STR_LIT>' ) <EOL> return False <EOL> if not self . _skip_consul : <EOL> if not self . _check_consul_for_service ( ) : <EOL> LOGGER . error ( '<STR_LIT>' ) <EOL> return False <EOL> LOGGER . info ( '<STR_LIT>' ) <EOL> self . _shutdown_other_versions ( ) <EOL> if self . _file_deployment : <EOL> self . _file_deployment . remove_other_archive_versions ( ) <EOL> parts = [ ] <EOL> if self . _group : <EOL> parts += [ self . _group ] <EOL> parts += [ self . _name , self . _version ] <EOL> LOGGER . info ( '<STR_LIT>' , <EOL> '<STR_LIT:U+0020>' . join ( parts ) ) <EOL> return True <EOL> def _deploy_shared_units ( self ) : <EOL> last_unit = self . _deployed_units [ - <NUM_LIT:1> ] <EOL> shared_unit_prefix = path . join ( self . _config_path , '<STR_LIT>' , '<STR_LIT>' ) <EOL> for name in self . _get_units ( ) : <EOL> version = None <EOL> if '<STR_LIT::>' in name : <EOL> name , version = name . split ( '<STR_LIT::>' ) <EOL> unit_file = path . join ( shared_unit_prefix , name ) <EOL> unit_name = utils . unit_name ( name , self . _name , <EOL> self . _group , <EOL> version ) <EOL> if not self . _deploy_unit ( unit_name , unit_file , last_unit ) : <EOL> LOGGER . error ( '<STR_LIT>' , unit_name ) <EOL> return False <EOL> last_unit = unit_name <EOL> return True <EOL> def _deploy_unit ( self , unit_name , unit_file , last_unit = None ) : <EOL> unit = self . _fleet . unit ( unit_name ) <EOL> unit . read_string ( self . _apply_variables ( self . _unit_file ( unit_file ) ) ) <EOL> if not self . _group : <EOL> for index , option in enumerate ( unit . options ( ) ) : <EOL> if option [ '<STR_LIT:name>' ] == '<STR_LIT>' and option [ '<STR_LIT:value>' ] . startswith ( '<STR_LIT>' ) : <EOL> LOGGER . debug ( '<STR_LIT>' ) <EOL> unit . _options . pop ( index ) <EOL> self . _maybe_add_last_unit ( unit , last_unit ) <EOL> if self . _unit_is_active ( unit_name ) : <EOL> self . _deployed_units . append ( unit_name ) <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name ) <EOL> return True <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name ) <EOL> if unit . submit ( ) : <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name ) <EOL> if unit . start ( ) : <EOL> if not self . _wait_for_unit_to_become_active ( unit_name ) : <EOL> LOGGER . error ( '<STR_LIT>' , unit_name ) <EOL> if not self . _no_removal : <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name ) <EOL> unit . destroy ( ) <EOL> return False <EOL> else : <EOL> LOGGER . error ( '<STR_LIT>' , unit_name ) <EOL> if not self . _no_removal : <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name ) <EOL> unit . destroy ( ) <EOL> return False <EOL> else : <EOL> LOGGER . error ( '<STR_LIT>' , unit_name ) <EOL> return False <EOL> LOGGER . info ( "<STR_LIT>" , unit_name ) <EOL> self . _deployed_units . append ( unit_name ) <EOL> return True <EOL> def _file_manifest ( self ) : <EOL> if self . _global : <EOL> file_path = path . join ( self . _config_path , '<STR_LIT>' , '<STR_LIT>' ) <EOL> else : <EOL> file_path = path . join ( self . _config_path , '<STR_LIT>' , self . _command , <EOL> '<STR_LIT>' . format ( self . _name ) ) <EOL> if not path . exists ( file_path ) : <EOL> return None <EOL> with open ( file_path ) as handle : <EOL> return handle . read ( ) <EOL> def _file_manifest_hash ( self ) : <EOL> value = self . _file_manifest ( ) <EOL> hash_value = hashlib . md5 ( value . encode ( '<STR_LIT:utf-8>' ) ) <EOL> return hash_value . hexdigest ( ) [ : <NUM_LIT:8> ] <EOL> def _get_units ( self ) : <EOL> units = [ ] <EOL> for unit in self . _config . get ( self . _command ) . get ( self . _name , [ ] ) : <EOL> match = UNIT_PATTERN . match ( unit ) <EOL> if match . group ( '<STR_LIT>' ) : <EOL> key , value = match . group ( '<STR_LIT>' ) . split ( '<STR_LIT::>' ) <EOL> if key == '<STR_LIT>' : <EOL> LOGGER . debug ( '<STR_LIT>' , <EOL> key , value , self . _environment ) <EOL> if value != self . _environment : <EOL> continue <EOL> if match . group ( '<STR_LIT:version>' ) : <EOL> units . append ( '<STR_LIT>' . format ( match . group ( '<STR_LIT:image>' ) , <EOL> match . group ( '<STR_LIT:version>' ) ) ) <EOL> else : <EOL> units . append ( match . group ( '<STR_LIT:image>' ) ) <EOL> return units <EOL> @ property <EOL> def _global ( self ) : <EOL> return self . _command == '<STR_LIT>' <EOL> def _load_config ( self , filename ) : <EOL> file = path . join ( self . _config_path , filename ) <EOL> if not path . exists ( file ) : <EOL> raise ValueError ( '<STR_LIT>' . format ( file ) ) <EOL> with open ( file , '<STR_LIT:r>' ) as handle : <EOL> return yaml . load ( handle ) <EOL> @ staticmethod <EOL> def _machine_label ( ntuple ) : <EOL> return '<STR_LIT>' . format ( ntuple . id [ <NUM_LIT:0> : <NUM_LIT:7> ] , ntuple . ipaddr ) <EOL> def _maybe_add_last_unit ( self , unit , last_unit ) : <EOL> if not last_unit or self . _no_dependencies : <EOL> return <EOL> for option in unit . options ( ) : <EOL> if ( option [ '<STR_LIT>' ] == '<STR_LIT>' and <EOL> option [ '<STR_LIT:name>' ] in [ '<STR_LIT>' , '<STR_LIT>' ] and <EOL> option [ '<STR_LIT:value>' ] == last_unit ) : <EOL> LOGGER . debug ( '<STR_LIT>' ) <EOL> return <EOL> options = unit . _options <EOL> unit . _options = [ ] <EOL> for option in options : <EOL> unit . add_option ( option . section , option . name , option . value ) <EOL> if option . section == '<STR_LIT>' and option . name == '<STR_LIT>' : <EOL> LOGGER . debug ( '<STR_LIT>' , last_unit ) <EOL> unit . add_option ( '<STR_LIT>' , '<STR_LIT>' , last_unit ) <EOL> unit . add_option ( '<STR_LIT>' , '<STR_LIT>' , last_unit ) <EOL> @ staticmethod <EOL> def _normalize_path ( value ) : <EOL> """<STR_LIT>""" <EOL> return path . abspath ( path . normpath ( value ) ) <EOL> def _remove_files ( self ) : <EOL> if self . _file_manifest ( ) : <EOL> unit_name = utils . unit_name ( '<STR_LIT>' , self . _name , self . _group , <EOL> self . _file_manifest_hash ( ) ) <EOL> self . _remove_unit ( unit_name ) <EOL> if self . _global : <EOL> manifest_file = '<STR_LIT>' <EOL> else : <EOL> manifest_file = '<STR_LIT>' . format ( self . _command , self . _name ) <EOL> file_deployment = files . FileDeployment ( unit_name , <EOL> self . env_config , <EOL> self . _config_path , <EOL> manifest_file , <EOL> self . _name , <EOL> self . _environment ) <EOL> file_deployment . remove_archive ( ) <EOL> file_deployment . remove_other_archive_versions ( ) <EOL> else : <EOL> LOGGER . debug ( '<STR_LIT>' ) <EOL> def _remove_globals ( self ) : <EOL> for name in self . _config . get ( '<STR_LIT>' , [ ] ) : <EOL> version = None <EOL> if '<STR_LIT::>' in name : <EOL> name , version = name . split ( '<STR_LIT::>' ) <EOL> self . _remove_unit ( utils . unit_name ( name , '<STR_LIT>' , None , version ) ) <EOL> def _remove_shared_units ( self ) : <EOL> for service in self . _get_units ( ) : <EOL> version = None <EOL> if '<STR_LIT::>' in service : <EOL> service , version = service . split ( '<STR_LIT::>' ) <EOL> self . _remove_unit ( utils . unit_name ( service , self . _name , self . _group , <EOL> version ) ) <EOL> def _remove_unit ( self , unit_name ) : <EOL> LOGGER . info ( '<STR_LIT>' , unit_name ) <EOL> unit = self . _fleet . unit ( unit_name ) <EOL> unit . destroy ( ) <EOL> def _remove_units ( self ) : <EOL> if self . _global : <EOL> self . _remove_files ( ) <EOL> self . _remove_globals ( ) <EOL> return True <EOL> self . _remove_unit ( utils . unit_name ( self . _name , None , self . _group , <EOL> self . _version ) ) <EOL> self . _remove_shared_units ( ) <EOL> self . _remove_files ( ) <EOL> return True <EOL> @ property <EOL> def _service ( self ) : <EOL> return self . _command == '<STR_LIT>' <EOL> def _shutdown_other_versions ( self ) : <EOL> LOGGER . debug ( '<STR_LIT>' ) <EOL> units = [ utils . parse_unit_name ( u . name ) for u in self . _fleet . units ( ) ] <EOL> destroy = set ( ) <EOL> for deployed_unit in self . _deployed_units : <EOL> parent , service , group , version = utils . parse_unit_name ( deployed_unit ) <EOL> for _parent , _service , _group , _version in units : <EOL> if _parent == parent and _service == service and _group == group and _version != version : <EOL> destroy . add ( ( _parent , _service , _group , _version ) ) <EOL> for name , version in destroy : <EOL> LOGGER . info ( '<STR_LIT>' , name , version ) <EOL> unit = self . _fleet . unit ( name , version ) <EOL> if not unit . destroy ( ) : <EOL> LOGGER . error ( '<STR_LIT>' , name , version ) <EOL> @ property <EOL> def _standalone ( self ) : <EOL> return self . _command == '<STR_LIT>' <EOL> def _unit_is_active ( self , unit_name , state = None ) : <EOL> state = self . _fleet . state ( True , unit_name ) if state is None else state <EOL> return state and all ( [ s . state == '<STR_LIT>' for s in state ] ) <EOL> def _unit_file ( self , name ) : <EOL> for extension in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> file_path = '<STR_LIT>' . format ( name , extension ) <EOL> if path . exists ( file_path ) : <EOL> with open ( file_path ) as handle : <EOL> if extension == '<STR_LIT>' : <EOL> return handle . read ( ) <EOL> data = yaml . load ( handle ) <EOL> if self . _global and '<STR_LIT>' in data : <EOL> return data [ '<STR_LIT>' ] <EOL> if self . _name in data : <EOL> return data [ self . _name ] <EOL> raise ValueError ( '<STR_LIT>' . format ( self . _name ) ) <EOL> raise ValueError ( '<STR_LIT>' . format ( name ) ) <EOL> def _wait_for_unit_to_become_active ( self , unit_name ) : <EOL> for attempt in range ( <NUM_LIT:0> , self . _max_tries ) : <EOL> state = self . _fleet . state ( True , unit_name ) <EOL> if self . _unit_is_active ( unit_name , state ) : <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name ) <EOL> return True <EOL> if state and all ( [ s . state == '<STR_LIT>' for s in state ] ) : <EOL> LOGGER . warn ( '<STR_LIT>' , unit_name ) <EOL> LOGGER . debug ( '<STR_LIT>' , state ) <EOL> return False <EOL> for s in [ s for s in state <EOL> if s . loaded and s . state == '<STR_LIT>' ] : <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name , <EOL> self . _machine_label ( s ) ) <EOL> for s in [ s for s in state if s . loaded and s . state == '<STR_LIT>' ] : <EOL> LOGGER . debug ( '<STR_LIT>' , unit_name , <EOL> self . _machine_label ( s ) ) <EOL> LOGGER . debug ( '<STR_LIT>' , <EOL> self . _delay ) <EOL> time . sleep ( self . _delay ) <EOL> LOGGER . warn ( '<STR_LIT>' , <EOL> self . _max_tries ) <EOL> return False </s>
