<s> import abc <EOL> from oslo_log import log as logging <EOL> import six <EOL> from sahara import conductor as cond <EOL> from sahara import context <EOL> from sahara . utils . openstack import nova <EOL> conductor = cond . API <EOL> LOG = logging . getLogger ( __name__ ) <EOL> @ six . add_metaclass ( abc . ABCMeta ) <EOL> class AutoConfigsProvider ( object ) : <EOL> def __init__ ( self , mapper , plugin_configs , cluster , scaling ) : <EOL> """<STR_LIT>""" <EOL> self . plugin_configs = plugin_configs <EOL> self . cluster = cluster <EOL> self . node_configs_to_update = mapper . get ( '<STR_LIT>' , { } ) <EOL> self . cluster_configs_to_update = mapper . get ( '<STR_LIT>' , { } ) <EOL> self . scaling = scaling <EOL> @ abc . abstractmethod <EOL> def _get_recommended_node_configs ( self , node_group ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> @ abc . abstractmethod <EOL> def _get_recommended_cluster_configs ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def _can_be_recommended ( self , configs_list , node_group = None ) : <EOL> """<STR_LIT>""" <EOL> cl_configs = self . cluster . cluster_configs . to_dict ( ) <EOL> for ncfg in configs_list : <EOL> section , name = self . _get_correct_section_and_name ( ncfg ) <EOL> if section in cl_configs and name in cl_configs [ section ] : <EOL> return False <EOL> if not node_group : <EOL> return True <EOL> cl_configs = node_group . node_configs . to_dict ( ) <EOL> for ncfg in configs_list : <EOL> section , name = self . _get_correct_section_and_name ( ncfg ) <EOL> if section in cl_configs and name in cl_configs [ section ] : <EOL> return False <EOL> return True <EOL> def _get_correct_section_and_name ( self , config_name ) : <EOL> """<STR_LIT>""" <EOL> section , name = None , None <EOL> if config_name in self . cluster_configs_to_update : <EOL> section = self . cluster_configs_to_update [ config_name ] [ <NUM_LIT:0> ] <EOL> name = self . cluster_configs_to_update [ config_name ] [ <NUM_LIT:1> ] <EOL> elif config_name in self . node_configs_to_update : <EOL> section = self . node_configs_to_update [ config_name ] [ <NUM_LIT:0> ] <EOL> name = self . node_configs_to_update [ config_name ] [ <NUM_LIT:1> ] <EOL> return section , name <EOL> def _get_default_config_value ( self , config_name ) : <EOL> """<STR_LIT>""" <EOL> section , name = self . _get_correct_section_and_name ( config_name ) <EOL> for config in self . plugin_configs : <EOL> if config . applicable_target == section and config . name == name : <EOL> return config . default_value <EOL> def _merge_configs ( self , current_configs , proposed_configs ) : <EOL> """<STR_LIT>""" <EOL> result = { } <EOL> for ( section , configs ) in six . iteritems ( proposed_configs ) : <EOL> cfg_values = { } <EOL> if section in current_configs : <EOL> cfg_values = ( current_configs [ section ] if <EOL> current_configs [ section ] else { } ) <EOL> cfg_values . update ( configs ) <EOL> result . update ( { section : cfg_values } ) <EOL> for ( section , configs ) in six . iteritems ( current_configs ) : <EOL> if section not in result : <EOL> result . update ( { section : configs } ) <EOL> return result <EOL> def _get_cluster_extra ( self ) : <EOL> cluster = self . cluster <EOL> return cluster . extra . to_dict ( ) if cluster . extra else { } <EOL> def finalize_autoconfiguration ( self ) : <EOL> if not self . cluster . use_autoconfig : <EOL> return <EOL> cluster_extra = self . _get_cluster_extra ( ) <EOL> cluster_extra [ '<STR_LIT>' ] = True <EOL> conductor . cluster_update ( <EOL> context . ctx ( ) , self . cluster , { '<STR_LIT>' : cluster_extra } ) <EOL> def apply_node_configs ( self , node_group ) : <EOL> """<STR_LIT>""" <EOL> if not node_group . use_autoconfig or not self . cluster . use_autoconfig : <EOL> return <EOL> to_update = self . node_configs_to_update <EOL> recommended_node_configs = self . _get_recommended_node_configs ( <EOL> node_group ) <EOL> if not recommended_node_configs : <EOL> return <EOL> current_dict = node_group . node_configs . to_dict ( ) <EOL> configuration = { } <EOL> for ncfg in six . iterkeys ( to_update ) : <EOL> if ncfg not in recommended_node_configs : <EOL> continue <EOL> n_section = to_update [ ncfg ] [ <NUM_LIT:0> ] <EOL> n_name = to_update [ ncfg ] [ <NUM_LIT:1> ] <EOL> proposed_config_value = recommended_node_configs [ ncfg ] <EOL> if n_section not in configuration : <EOL> configuration . update ( { n_section : { } } ) <EOL> configuration [ n_section ] . update ( { n_name : proposed_config_value } ) <EOL> current_dict = self . _merge_configs ( current_dict , configuration ) <EOL> conductor . node_group_update ( context . ctx ( ) , node_group , <EOL> { '<STR_LIT>' : current_dict } ) <EOL> def apply_cluster_configs ( self ) : <EOL> """<STR_LIT>""" <EOL> cluster = self . cluster <EOL> if not cluster . use_autoconfig : <EOL> return <EOL> to_update = self . cluster_configs_to_update <EOL> recommended_cluster_configs = self . _get_recommended_cluster_configs ( ) <EOL> if not recommended_cluster_configs : <EOL> return <EOL> current_dict = cluster . cluster_configs . to_dict ( ) <EOL> configuration = { } <EOL> for ncfg in six . iterkeys ( to_update ) : <EOL> if ncfg not in recommended_cluster_configs : <EOL> continue <EOL> n_section = to_update [ ncfg ] [ <NUM_LIT:0> ] <EOL> n_name = to_update [ ncfg ] [ <NUM_LIT:1> ] <EOL> proposed_config_value = recommended_cluster_configs [ ncfg ] <EOL> if n_section not in configuration : <EOL> configuration . update ( { n_section : { } } ) <EOL> configuration [ n_section ] . update ( { n_name : proposed_config_value } ) <EOL> current_dict = self . _merge_configs ( current_dict , configuration ) <EOL> conductor . cluster_update ( context . ctx ( ) , cluster , <EOL> { '<STR_LIT>' : current_dict } ) <EOL> def apply_recommended_configs ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . scaling : <EOL> cluster_extra = self . _get_cluster_extra ( ) <EOL> if '<STR_LIT>' not in cluster_extra : <EOL> return <EOL> for ng in self . cluster . node_groups : <EOL> self . apply_node_configs ( ng ) <EOL> self . apply_cluster_configs ( ) <EOL> configs = list ( self . cluster_configs_to_update . keys ( ) ) <EOL> configs . extend ( list ( self . node_configs_to_update . keys ( ) ) ) <EOL> LOG . debug ( "<STR_LIT>" . format ( <EOL> configs = configs ) ) <EOL> self . finalize_autoconfiguration ( ) <EOL> class HadoopAutoConfigsProvider ( AutoConfigsProvider ) : <EOL> def __init__ ( self , mapper , plugin_configs , cluster , scaling , hbase = False ) : <EOL> super ( HadoopAutoConfigsProvider , self ) . __init__ ( <EOL> mapper , plugin_configs , cluster , scaling ) <EOL> self . requested_flavors = { } <EOL> self . is_hbase_enabled = hbase <EOL> def _get_java_opts ( self , value ) : <EOL> return "<STR_LIT>" % int ( value ) <EOL> def _transform_mb_to_gb ( self , mb ) : <EOL> return mb / <NUM_LIT> <EOL> def _transform_gb_to_mb ( self , gb ) : <EOL> return gb * <NUM_LIT> <EOL> def _get_min_size_of_container ( self , ram ) : <EOL> if ram <= <NUM_LIT:4> : <EOL> return <NUM_LIT> <EOL> if ram <= <NUM_LIT:8> : <EOL> return <NUM_LIT> <EOL> if ram <= <NUM_LIT> : <EOL> return <NUM_LIT> <EOL> return <NUM_LIT> <EOL> def _get_os_ram_recommendation ( self , ram ) : <EOL> upper_bounds = [ <NUM_LIT:4> , <NUM_LIT:8> , <NUM_LIT:16> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT:64> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] <EOL> reserve_for_os = [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT:6> , <NUM_LIT:8> , <NUM_LIT:8> , <NUM_LIT:12> , <NUM_LIT> , <NUM_LIT:32> ] <EOL> for ( upper , reserve ) in zip ( upper_bounds , reserve_for_os ) : <EOL> if ram <= upper : <EOL> return reserve <EOL> return <NUM_LIT:64> <EOL> def _get_hbase_ram_recommendations ( self , ram ) : <EOL> if not self . is_hbase_enabled : <EOL> return <NUM_LIT:0> <EOL> upper_bounds = [ <NUM_LIT:4> , <NUM_LIT:8> , <NUM_LIT:16> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT:64> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] <EOL> reserve_for_hbase = [ <NUM_LIT:1> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:4> , <NUM_LIT:8> , <NUM_LIT:8> , <NUM_LIT:8> , <NUM_LIT:16> , <NUM_LIT> , <NUM_LIT:32> ] <EOL> for ( upper , reserve ) in zip ( upper_bounds , reserve_for_hbase ) : <EOL> if ram <= upper : <EOL> return reserve <EOL> return <NUM_LIT:64> <EOL> def _get_node_group_data ( self , node_group ) : <EOL> if node_group . flavor_id not in self . requested_flavors : <EOL> flavor = nova . get_flavor ( id = node_group . flavor_id ) <EOL> self . requested_flavors [ node_group . flavor_id ] = flavor <EOL> else : <EOL> flavor = self . requested_flavors [ node_group . flavor_id ] <EOL> cpu = flavor . vcpus <EOL> ram = flavor . ram <EOL> data = { } <EOL> os = self . _get_os_ram_recommendation ( self . _transform_mb_to_gb ( ram ) ) <EOL> hbase = self . _get_hbase_ram_recommendations ( <EOL> self . _transform_mb_to_gb ( ram ) ) <EOL> reserved_ram = self . _transform_gb_to_mb ( os + hbase ) <EOL> min_container_size = self . _get_min_size_of_container ( <EOL> self . _transform_mb_to_gb ( ram ) ) <EOL> data [ "<STR_LIT>" ] = int ( max ( <EOL> <NUM_LIT:8> , min ( <NUM_LIT:2> * cpu , ram / min_container_size ) ) ) <EOL> data [ "<STR_LIT>" ] = ( ram - reserved_ram ) / data [ "<STR_LIT>" ] <EOL> data [ "<STR_LIT>" ] = max ( data [ "<STR_LIT>" ] , <EOL> min_container_size ) <EOL> data [ "<STR_LIT>" ] = min ( <NUM_LIT> , int ( data [ "<STR_LIT>" ] ) ) <EOL> data [ "<STR_LIT>" ] = int ( data [ "<STR_LIT>" ] ) <EOL> data [ "<STR_LIT>" ] = int ( data [ "<STR_LIT>" ] ) <EOL> data [ "<STR_LIT>" ] = int ( <NUM_LIT:2> * data [ "<STR_LIT>" ] ) <EOL> data [ "<STR_LIT>" ] = int ( min ( data [ "<STR_LIT>" ] , data [ "<STR_LIT>" ] ) ) <EOL> return data <EOL> def _get_recommended_node_configs ( self , node_group ) : <EOL> """<STR_LIT>""" <EOL> configs_to_update = list ( self . node_configs_to_update . keys ( ) ) <EOL> if not self . _can_be_recommended ( configs_to_update , node_group ) : <EOL> return { } <EOL> data = self . _get_node_group_data ( node_group ) <EOL> r = { } <EOL> r [ '<STR_LIT>' ] = ( data [ '<STR_LIT>' ] * <EOL> data [ '<STR_LIT>' ] ) <EOL> r [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] <EOL> r [ '<STR_LIT>' ] = ( data [ '<STR_LIT>' ] * <EOL> data [ '<STR_LIT>' ] ) <EOL> r [ '<STR_LIT>' ] = "<STR_LIT:false>" <EOL> r [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] <EOL> r [ '<STR_LIT>' ] = self . _get_java_opts ( <EOL> <NUM_LIT> * data [ '<STR_LIT>' ] ) <EOL> r [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] <EOL> r [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] <EOL> r [ '<STR_LIT>' ] = self . _get_java_opts ( <EOL> <NUM_LIT> * data [ '<STR_LIT>' ] ) <EOL> r [ '<STR_LIT>' ] = self . _get_java_opts ( <EOL> <NUM_LIT> * data [ '<STR_LIT>' ] ) <EOL> r [ '<STR_LIT>' ] = int ( min ( <EOL> <NUM_LIT> * data [ '<STR_LIT>' ] , <NUM_LIT> ) ) <EOL> return r <EOL> def get_datanode_name ( self ) : <EOL> return "<STR_LIT>" <EOL> def _get_recommended_cluster_configs ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . _can_be_recommended ( [ '<STR_LIT>' ] ) : <EOL> return { } <EOL> datanode_count = <NUM_LIT:0> <EOL> datanode_proc_name = self . get_datanode_name ( ) <EOL> for ng in self . cluster . node_groups : <EOL> if datanode_proc_name in ng . node_processes : <EOL> datanode_count += ng . count <EOL> replica = '<STR_LIT>' <EOL> recommended_value = self . _get_default_config_value ( replica ) <EOL> if recommended_value : <EOL> return { replica : min ( recommended_value , datanode_count ) } <EOL> else : <EOL> return { } </s>
