<s> from sahara . i18n import _ <EOL> from sahara . plugins . cdh import cloudera_utils as cu <EOL> from sahara . plugins . cdh . v5_3_0 import config_helper <EOL> from sahara . plugins . cdh . v5_3_0 import plugin_utils as pu <EOL> from sahara . plugins . cdh . v5_3_0 import validation <EOL> from sahara . swift import swift_helper <EOL> from sahara . utils import cluster_progress_ops as cpo <EOL> from sahara . utils import configs as s_cfg <EOL> from sahara . utils import xmlutils <EOL> HDFS_SERVICE_TYPE = '<STR_LIT>' <EOL> YARN_SERVICE_TYPE = '<STR_LIT>' <EOL> OOZIE_SERVICE_TYPE = '<STR_LIT>' <EOL> HIVE_SERVICE_TYPE = '<STR_LIT>' <EOL> HUE_SERVICE_TYPE = '<STR_LIT>' <EOL> SPARK_SERVICE_TYPE = '<STR_LIT>' <EOL> ZOOKEEPER_SERVICE_TYPE = '<STR_LIT>' <EOL> HBASE_SERVICE_TYPE = '<STR_LIT>' <EOL> FLUME_SERVICE_TYPE = '<STR_LIT>' <EOL> SENTRY_SERVICE_TYPE = '<STR_LIT>' <EOL> SOLR_SERVICE_TYPE = '<STR_LIT>' <EOL> SQOOP_SERVICE_TYPE = '<STR_LIT>' <EOL> KS_INDEXER_SERVICE_TYPE = '<STR_LIT>' <EOL> IMPALA_SERVICE_TYPE = '<STR_LIT>' <EOL> c_helper = config_helper . ConfigHelperV530 ( ) <EOL> class ClouderaUtilsV530 ( cu . ClouderaUtils ) : <EOL> FLUME_SERVICE_NAME = '<STR_LIT>' <EOL> SOLR_SERVICE_NAME = '<STR_LIT>' <EOL> SQOOP_SERVICE_NAME = '<STR_LIT>' <EOL> KS_INDEXER_SERVICE_NAME = '<STR_LIT>' <EOL> IMPALA_SERVICE_NAME = '<STR_LIT>' <EOL> SENTRY_SERVICE_NAME = '<STR_LIT>' <EOL> CM_API_VERSION = <NUM_LIT:8> <EOL> def __init__ ( self ) : <EOL> cu . ClouderaUtils . __init__ ( self ) <EOL> self . pu = pu . PluginUtilsV530 ( ) <EOL> self . validator = validation . ValidatorV530 <EOL> def get_service_by_role ( self , role , cluster = None , instance = None ) : <EOL> cm_cluster = None <EOL> if cluster : <EOL> cm_cluster = self . get_cloudera_cluster ( cluster ) <EOL> elif instance : <EOL> cm_cluster = self . get_cloudera_cluster ( instance . cluster ) <EOL> else : <EOL> raise ValueError ( _ ( "<STR_LIT>" ) ) <EOL> if role in [ '<STR_LIT>' ] : <EOL> return cm_cluster . get_service ( self . FLUME_SERVICE_NAME ) <EOL> elif role in [ '<STR_LIT>' ] : <EOL> return cm_cluster . get_service ( self . SENTRY_SERVICE_NAME ) <EOL> elif role in [ '<STR_LIT>' ] : <EOL> return cm_cluster . get_service ( self . SQOOP_SERVICE_NAME ) <EOL> elif role in [ '<STR_LIT>' ] : <EOL> return cm_cluster . get_service ( self . SOLR_SERVICE_NAME ) <EOL> elif role in [ '<STR_LIT>' ] : <EOL> return cm_cluster . get_service ( self . KS_INDEXER_SERVICE_NAME ) <EOL> elif role in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> return cm_cluster . get_service ( self . IMPALA_SERVICE_NAME ) <EOL> else : <EOL> return super ( ClouderaUtilsV530 , self ) . get_service_by_role ( <EOL> role , cluster , instance ) <EOL> @ cpo . event_wrapper ( <EOL> True , step = _ ( "<STR_LIT>" ) , param = ( '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> @ cu . cloudera_cmd <EOL> def first_run ( self , cluster ) : <EOL> cm_cluster = self . get_cloudera_cluster ( cluster ) <EOL> yield cm_cluster . first_run ( ) <EOL> @ cpo . event_wrapper ( True , step = _ ( "<STR_LIT>" ) , param = ( '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> def create_services ( self , cluster ) : <EOL> api = self . get_api_client ( cluster ) <EOL> cm_cluster = api . create_cluster ( cluster . name , <EOL> fullVersion = cluster . hadoop_version ) <EOL> if len ( self . pu . get_zookeepers ( cluster ) ) > <NUM_LIT:0> : <EOL> cm_cluster . create_service ( self . ZOOKEEPER_SERVICE_NAME , <EOL> ZOOKEEPER_SERVICE_TYPE ) <EOL> cm_cluster . create_service ( self . HDFS_SERVICE_NAME , HDFS_SERVICE_TYPE ) <EOL> cm_cluster . create_service ( self . YARN_SERVICE_NAME , YARN_SERVICE_TYPE ) <EOL> cm_cluster . create_service ( self . OOZIE_SERVICE_NAME , OOZIE_SERVICE_TYPE ) <EOL> if self . pu . get_hive_metastore ( cluster ) : <EOL> cm_cluster . create_service ( self . HIVE_SERVICE_NAME , <EOL> HIVE_SERVICE_TYPE ) <EOL> if self . pu . get_hue ( cluster ) : <EOL> cm_cluster . create_service ( self . HUE_SERVICE_NAME , HUE_SERVICE_TYPE ) <EOL> if self . pu . get_spark_historyserver ( cluster ) : <EOL> cm_cluster . create_service ( self . SPARK_SERVICE_NAME , <EOL> SPARK_SERVICE_TYPE ) <EOL> if self . pu . get_hbase_master ( cluster ) : <EOL> cm_cluster . create_service ( self . HBASE_SERVICE_NAME , <EOL> HBASE_SERVICE_TYPE ) <EOL> if len ( self . pu . get_flumes ( cluster ) ) > <NUM_LIT:0> : <EOL> cm_cluster . create_service ( self . FLUME_SERVICE_NAME , <EOL> FLUME_SERVICE_TYPE ) <EOL> if self . pu . get_sentry ( cluster ) : <EOL> cm_cluster . create_service ( self . SENTRY_SERVICE_NAME , <EOL> SENTRY_SERVICE_TYPE ) <EOL> if len ( self . pu . get_solrs ( cluster ) ) > <NUM_LIT:0> : <EOL> cm_cluster . create_service ( self . SOLR_SERVICE_NAME , <EOL> SOLR_SERVICE_TYPE ) <EOL> if self . pu . get_sqoop ( cluster ) : <EOL> cm_cluster . create_service ( self . SQOOP_SERVICE_NAME , <EOL> SQOOP_SERVICE_TYPE ) <EOL> if len ( self . pu . get_hbase_indexers ( cluster ) ) > <NUM_LIT:0> : <EOL> cm_cluster . create_service ( self . KS_INDEXER_SERVICE_NAME , <EOL> KS_INDEXER_SERVICE_TYPE ) <EOL> if self . pu . get_catalogserver ( cluster ) : <EOL> cm_cluster . create_service ( self . IMPALA_SERVICE_NAME , <EOL> IMPALA_SERVICE_TYPE ) <EOL> def await_agents ( self , cluster , instances ) : <EOL> self . _await_agents ( cluster , instances , c_helper . AWAIT_AGENTS_TIMEOUT ) <EOL> @ cpo . event_wrapper ( <EOL> True , step = _ ( "<STR_LIT>" ) , param = ( '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> def configure_services ( self , cluster ) : <EOL> cm_cluster = self . get_cloudera_cluster ( cluster ) <EOL> if len ( self . pu . get_zookeepers ( cluster ) ) > <NUM_LIT:0> : <EOL> zookeeper = cm_cluster . get_service ( self . ZOOKEEPER_SERVICE_NAME ) <EOL> zookeeper . update_config ( self . _get_configs ( ZOOKEEPER_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> hdfs = cm_cluster . get_service ( self . HDFS_SERVICE_NAME ) <EOL> hdfs . update_config ( self . _get_configs ( HDFS_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> yarn = cm_cluster . get_service ( self . YARN_SERVICE_NAME ) <EOL> yarn . update_config ( self . _get_configs ( YARN_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> oozie = cm_cluster . get_service ( self . OOZIE_SERVICE_NAME ) <EOL> oozie . update_config ( self . _get_configs ( OOZIE_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if self . pu . get_hive_metastore ( cluster ) : <EOL> hive = cm_cluster . get_service ( self . HIVE_SERVICE_NAME ) <EOL> hive . update_config ( self . _get_configs ( HIVE_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if self . pu . get_hue ( cluster ) : <EOL> hue = cm_cluster . get_service ( self . HUE_SERVICE_NAME ) <EOL> hue . update_config ( self . _get_configs ( HUE_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if self . pu . get_spark_historyserver ( cluster ) : <EOL> spark = cm_cluster . get_service ( self . SPARK_SERVICE_NAME ) <EOL> spark . update_config ( self . _get_configs ( SPARK_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if self . pu . get_hbase_master ( cluster ) : <EOL> hbase = cm_cluster . get_service ( self . HBASE_SERVICE_NAME ) <EOL> hbase . update_config ( self . _get_configs ( HBASE_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if len ( self . pu . get_flumes ( cluster ) ) > <NUM_LIT:0> : <EOL> flume = cm_cluster . get_service ( self . FLUME_SERVICE_NAME ) <EOL> flume . update_config ( self . _get_configs ( FLUME_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if self . pu . get_sentry ( cluster ) : <EOL> sentry = cm_cluster . get_service ( self . SENTRY_SERVICE_NAME ) <EOL> sentry . update_config ( self . _get_configs ( SENTRY_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if len ( self . pu . get_solrs ( cluster ) ) > <NUM_LIT:0> : <EOL> solr = cm_cluster . get_service ( self . SOLR_SERVICE_NAME ) <EOL> solr . update_config ( self . _get_configs ( SOLR_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if self . pu . get_sqoop ( cluster ) : <EOL> sqoop = cm_cluster . get_service ( self . SQOOP_SERVICE_NAME ) <EOL> sqoop . update_config ( self . _get_configs ( SQOOP_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> if len ( self . pu . get_hbase_indexers ( cluster ) ) > <NUM_LIT:0> : <EOL> ks_indexer = cm_cluster . get_service ( self . KS_INDEXER_SERVICE_NAME ) <EOL> ks_indexer . update_config ( <EOL> self . _get_configs ( KS_INDEXER_SERVICE_TYPE , cluster = cluster ) ) <EOL> if self . pu . get_catalogserver ( cluster ) : <EOL> impala = cm_cluster . get_service ( self . IMPALA_SERVICE_NAME ) <EOL> impala . update_config ( self . _get_configs ( IMPALA_SERVICE_TYPE , <EOL> cluster = cluster ) ) <EOL> def _get_configs ( self , service , cluster = None , instance = None ) : <EOL> def get_hadoop_dirs ( mount_points , suffix ) : <EOL> return '<STR_LIT:U+002C>' . join ( [ x + suffix for x in mount_points ] ) <EOL> all_confs = { } <EOL> if cluster : <EOL> zk_count = self . validator . _get_inst_count ( cluster , <EOL> '<STR_LIT>' ) <EOL> hbm_count = self . validator . _get_inst_count ( cluster , '<STR_LIT>' ) <EOL> snt_count = self . validator . _get_inst_count ( cluster , <EOL> '<STR_LIT>' ) <EOL> ks_count = self . validator . _get_inst_count ( cluster , <EOL> '<STR_LIT>' ) <EOL> imp_count = self . validator . _get_inst_count ( cluster , <EOL> '<STR_LIT>' ) <EOL> hive_count = self . validator . _get_inst_count ( cluster , <EOL> '<STR_LIT>' ) <EOL> slr_count = self . validator . _get_inst_count ( cluster , '<STR_LIT>' ) <EOL> sqp_count = self . validator . _get_inst_count ( cluster , '<STR_LIT>' ) <EOL> core_site_safety_valve = '<STR_LIT>' <EOL> if self . pu . c_helper . is_swift_enabled ( cluster ) : <EOL> configs = swift_helper . get_swift_configs ( ) <EOL> confs = { c [ '<STR_LIT:name>' ] : c [ '<STR_LIT:value>' ] for c in configs } <EOL> core_site_safety_valve = xmlutils . create_elements_xml ( confs ) <EOL> all_confs = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : <EOL> self . ZOOKEEPER_SERVICE_NAME if zk_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> '<STR_LIT>' if imp_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : core_site_safety_valve <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . YARN_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . SENTRY_SERVICE_NAME if snt_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . ZOOKEEPER_SERVICE_NAME if zk_count else '<STR_LIT>' <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . YARN_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . HIVE_SERVICE_NAME if hive_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . ZOOKEEPER_SERVICE_NAME if zk_count else '<STR_LIT>' <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HDFS_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . ZOOKEEPER_SERVICE_NAME if zk_count else '<STR_LIT>' <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HIVE_SERVICE_NAME , <EOL> '<STR_LIT>' : self . OOZIE_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . SENTRY_SERVICE_NAME if snt_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . SOLR_SERVICE_NAME if slr_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . ZOOKEEPER_SERVICE_NAME if zk_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . HBASE_SERVICE_NAME if hbm_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . IMPALA_SERVICE_NAME if imp_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . SQOOP_SERVICE_NAME if sqp_count else '<STR_LIT>' <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . YARN_SERVICE_NAME <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HDFS_SERVICE_NAME , <EOL> '<STR_LIT>' : self . ZOOKEEPER_SERVICE_NAME , <EOL> '<STR_LIT>' : '<STR_LIT:true>' if ks_count else '<STR_LIT:false>' , <EOL> '<STR_LIT>' : <EOL> '<STR_LIT:true>' if ks_count else '<STR_LIT:false>' <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HDFS_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . SOLR_SERVICE_NAME if slr_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . HBASE_SERVICE_NAME if hbm_count else '<STR_LIT>' <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HDFS_SERVICE_NAME , <EOL> '<STR_LIT>' : ( <EOL> c_helper . SENTRY_IMPALA_CLIENT_SAFETY_VALVE <EOL> if imp_count else '<STR_LIT>' ) <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HDFS_SERVICE_NAME , <EOL> '<STR_LIT>' : self . ZOOKEEPER_SERVICE_NAME <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . YARN_SERVICE_NAME <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HBASE_SERVICE_NAME , <EOL> '<STR_LIT>' : self . SOLR_SERVICE_NAME <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . HDFS_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . HBASE_SERVICE_NAME if hbm_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : self . HIVE_SERVICE_NAME , <EOL> '<STR_LIT>' : <EOL> self . SENTRY_SERVICE_NAME if snt_count else '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . ZOOKEEPER_SERVICE_NAME if zk_count else '<STR_LIT>' <EOL> } <EOL> } <EOL> hive_confs = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . pu . get_manager ( cluster ) . internal_ip , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . pu . db_helper . get_hive_db_password ( cluster ) <EOL> } <EOL> } <EOL> hue_confs = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : self . pu . get_role_name ( <EOL> self . pu . get_namenode ( cluster ) , '<STR_LIT>' ) <EOL> } <EOL> } <EOL> sentry_confs = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . pu . get_manager ( cluster ) . internal_ip , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : <EOL> self . pu . db_helper . get_sentry_db_password ( cluster ) <EOL> } <EOL> } <EOL> all_confs = s_cfg . merge_configs ( all_confs , hue_confs ) <EOL> all_confs = s_cfg . merge_configs ( all_confs , hive_confs ) <EOL> all_confs = s_cfg . merge_configs ( all_confs , sentry_confs ) <EOL> all_confs = s_cfg . merge_configs ( all_confs , cluster . cluster_configs ) <EOL> if instance : <EOL> paths = instance . storage_paths ( ) <EOL> instance_default_confs = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : get_hadoop_dirs ( paths , '<STR_LIT>' ) <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : <EOL> get_hadoop_dirs ( paths , '<STR_LIT>' ) <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : get_hadoop_dirs ( paths , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT:30> <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : <EOL> get_hadoop_dirs ( paths , '<STR_LIT>' ) <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : <NUM_LIT> <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : <EOL> '<STR_LIT:false>' if snt_count else '<STR_LIT:true>' , <EOL> '<STR_LIT>' : ( <EOL> c_helper . HIVE_SERVER2_SENTRY_SAFETY_VALVE <EOL> if snt_count else '<STR_LIT>' ) <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : ( <EOL> c_helper . HIVE_METASTORE_SENTRY_SAFETY_VALVE <EOL> if snt_count else '<STR_LIT>' ) <EOL> } <EOL> } <EOL> ng_user_confs = self . pu . convert_process_configs ( <EOL> instance . node_group . node_configs ) <EOL> all_confs = s_cfg . merge_configs ( all_confs , ng_user_confs ) <EOL> all_confs = s_cfg . merge_configs ( all_confs , instance_default_confs ) <EOL> return all_confs . get ( service , { } ) </s>
