<s> import argparse <EOL> import deploy_utils <EOL> import deploy_zookeeper <EOL> import os <EOL> import parallel_deploy <EOL> import pwd <EOL> import socket <EOL> import subprocess <EOL> import sys <EOL> import tempfile <EOL> import urlparse <EOL> from log import Log <EOL> ALL_JOBS = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> SHELL_COMMAND_INFO = { <EOL> "<STR_LIT>" : ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) , <EOL> "<STR_LIT>" : ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> "<STR_LIT:version>" : ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> } <EOL> def generate_metrics_config ( args , host , job_name , instance_id = - <NUM_LIT:1> ) : <EOL> job = args . hbase_config . jobs [ job_name ] <EOL> supervisor_client = deploy_utils . get_supervisor_client ( host , <EOL> "<STR_LIT>" , args . hbase_config . cluster . name , job_name , instance_id = instance_id ) <EOL> ganglia_switch = "<STR_LIT>" <EOL> if args . hbase_config . cluster . ganglia_address : <EOL> ganglia_switch = "<STR_LIT>" <EOL> config_dict = { <EOL> "<STR_LIT>" : job_name , <EOL> "<STR_LIT>" : <NUM_LIT:10> , <EOL> "<STR_LIT>" : supervisor_client . get_log_dir ( ) , <EOL> "<STR_LIT>" : args . hbase_config . cluster . ganglia_address , <EOL> "<STR_LIT>" : ganglia_switch , <EOL> } <EOL> local_path = "<STR_LIT>" % deploy_utils . get_template_dir ( ) <EOL> template = deploy_utils . Template ( open ( local_path , "<STR_LIT:r>" ) . read ( ) ) <EOL> return template . substitute ( config_dict ) <EOL> def generate_zk_jaas_config ( args ) : <EOL> if not deploy_utils . is_security_enabled ( args ) : <EOL> return "<STR_LIT>" <EOL> config_dict = args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] <EOL> for key , value in config_dict . items ( ) [ <NUM_LIT:1> : ] : <EOL> if value != "<STR_LIT:true>" and value != "<STR_LIT:false>" and value . find ( "<STR_LIT>" ) == - <NUM_LIT:1> : <EOL> config_dict [ key ] = "<STR_LIT>" + value + "<STR_LIT>" <EOL> header_line = config_dict [ "<STR_LIT>" ] <EOL> return "<STR_LIT>" % ( header_line , <EOL> "<STR_LIT:\n>" . join ( [ "<STR_LIT>" % ( key , value ) <EOL> for ( key , value ) in config_dict . iteritems ( ) if key != config_dict . keys ( ) [ <NUM_LIT:0> ] ] ) ) <EOL> def generate_configs ( args , host , job_name , instance_id ) : <EOL> core_site_xml = deploy_utils . generate_site_xml ( args , <EOL> args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] ) <EOL> hdfs_site_xml = deploy_utils . generate_site_xml ( args , <EOL> args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] ) <EOL> hbase_site_xml = deploy_utils . generate_site_xml ( args , <EOL> args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] ) <EOL> hadoop_metrics_properties = generate_metrics_config ( args , host , job_name , instance_id ) <EOL> zk_jaas_conf = generate_zk_jaas_config ( args ) <EOL> config_files = { <EOL> "<STR_LIT>" : core_site_xml , <EOL> "<STR_LIT>" : hdfs_site_xml , <EOL> "<STR_LIT>" : hbase_site_xml , <EOL> "<STR_LIT>" : hadoop_metrics_properties , <EOL> "<STR_LIT>" : zk_jaas_conf , <EOL> } <EOL> config_files . update ( args . hbase_config . configuration . raw_files ) <EOL> return config_files <EOL> def generate_run_scripts_params ( args , host , job_name , host_id , instance_id ) : <EOL> job = args . hbase_config . jobs [ job_name ] <EOL> supervisor_client = deploy_utils . get_supervisor_client ( host , <EOL> "<STR_LIT>" , args . hbase_config . cluster . name , job_name , instance_id = instance_id ) <EOL> artifact_and_version = "<STR_LIT>" + args . hbase_config . cluster . version <EOL> component_dir = "<STR_LIT>" <EOL> jar_dirs = "<STR_LIT>" % ( component_dir , component_dir , component_dir ) <EOL> log_level = deploy_utils . get_service_log_level ( args , args . hbase_config ) <EOL> params = job . get_arguments ( args , args . hbase_config . cluster , args . hbase_config . jobs , <EOL> args . hbase_config . arguments_dict , job_name , host_id , instance_id ) <EOL> script_dict = { <EOL> "<STR_LIT>" : artifact_and_version , <EOL> "<STR_LIT>" : job_name , <EOL> "<STR_LIT>" : jar_dirs , <EOL> "<STR_LIT>" : supervisor_client . get_run_dir ( ) , <EOL> "<STR_LIT>" : params , <EOL> } <EOL> return script_dict <EOL> def get_hbase_service_config ( args ) : <EOL> args . hbase_config = deploy_utils . get_service_config ( args ) <EOL> if not args . hbase_config . cluster . zk_cluster : <EOL> Log . print_critical ( <EOL> "<STR_LIT>" % <EOL> args . hbase_config . cluster . name ) <EOL> def generate_start_script ( args , host , job_name , host_id , instance_id ) : <EOL> script_params = generate_run_scripts_params ( args , host , job_name , host_id , instance_id ) <EOL> return deploy_utils . create_run_script ( <EOL> "<STR_LIT>" % deploy_utils . get_template_dir ( ) , <EOL> script_params ) <EOL> def install ( args ) : <EOL> get_hbase_service_config ( args ) <EOL> deploy_utils . install_service ( args , "<STR_LIT>" , args . hbase_config , "<STR_LIT>" ) <EOL> def cleanup_job ( args , host , job_name , host_id , instance_id , cleanup_token , active ) : <EOL> deploy_utils . cleanup_job ( "<STR_LIT>" , args . hbase_config , <EOL> host , job_name , instance_id , cleanup_token ) <EOL> def cleanup ( args ) : <EOL> get_hbase_service_config ( args ) <EOL> cleanup_token = deploy_utils . confirm_cleanup ( args , <EOL> "<STR_LIT>" , args . hbase_config ) <EOL> for job_name in args . job or ALL_JOBS : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , <EOL> '<STR_LIT>' , cleanup_token = cleanup_token ) <EOL> parallel_deploy . start_deploy_threads ( cleanup_job , task_list ) <EOL> def bootstrap_job ( args , host , job_name , host_id , instance_id , cleanup_token , active ) : <EOL> args . hbase_config . parse_generated_config_files ( args , job_name , host_id , instance_id ) <EOL> deploy_utils . bootstrap_job ( args , "<STR_LIT>" , "<STR_LIT>" , <EOL> args . hbase_config , host , job_name , instance_id , cleanup_token , '<STR_LIT:0>' ) <EOL> start_job ( args , host , job_name , host_id , instance_id ) <EOL> def bootstrap ( args ) : <EOL> get_hbase_service_config ( args ) <EOL> cleanup_token = deploy_utils . confirm_bootstrap ( "<STR_LIT>" , args . hbase_config ) <EOL> for job_name in args . job or ALL_JOBS : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , <EOL> '<STR_LIT>' , cleanup_token = cleanup_token ) <EOL> parallel_deploy . start_deploy_threads ( bootstrap_job , task_list ) <EOL> def start_job ( args , host , job_name , host_id , instance_id , is_wait = False ) : <EOL> if is_wait : <EOL> deploy_utils . wait_for_job_stopping ( "<STR_LIT>" , <EOL> args . hbase_config . cluster . name , job_name , host , instance_id ) <EOL> args . hbase_config . parse_generated_config_files ( args , job_name , host_id , instance_id ) <EOL> config_files = generate_configs ( args , host , job_name , instance_id ) <EOL> start_script = generate_start_script ( args , host , job_name , host_id , instance_id ) <EOL> http_url = deploy_utils . get_http_service_uri ( host , <EOL> args . hbase_config . jobs [ job_name ] . base_port , instance_id ) <EOL> deploy_utils . start_job ( args , "<STR_LIT>" , "<STR_LIT>" , args . hbase_config , <EOL> host , job_name , instance_id , start_script , http_url , ** config_files ) <EOL> def start ( args ) : <EOL> if not args . skip_confirm : <EOL> deploy_utils . confirm_start ( args ) <EOL> get_hbase_service_config ( args ) <EOL> for job_name in args . job or ALL_JOBS : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , '<STR_LIT:start>' ) <EOL> parallel_deploy . start_deploy_threads ( start_job , task_list ) <EOL> def stop_job ( args , host , job_name , instance_id ) : <EOL> deploy_utils . stop_job ( "<STR_LIT>" , args . hbase_config , <EOL> host , job_name , instance_id ) <EOL> def stop ( args ) : <EOL> if not args . skip_confirm : <EOL> deploy_utils . confirm_stop ( args ) <EOL> get_hbase_service_config ( args ) <EOL> for job_name in args . job or reversed ( ALL_JOBS ) : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , '<STR_LIT>' ) <EOL> parallel_deploy . start_deploy_threads ( stop_job , task_list ) <EOL> def restart ( args ) : <EOL> if not args . skip_confirm : <EOL> deploy_utils . confirm_restart ( args ) <EOL> get_hbase_service_config ( args ) <EOL> for job_name in args . job or reversed ( ALL_JOBS ) : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , '<STR_LIT>' ) <EOL> parallel_deploy . start_deploy_threads ( stop_job , task_list ) <EOL> for job_name in args . job or ALL_JOBS : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , <EOL> '<STR_LIT:start>' , is_wait = True ) <EOL> parallel_deploy . start_deploy_threads ( start_job , task_list ) <EOL> def show_job ( args , host , job_name , instance_id ) : <EOL> deploy_utils . show_job ( "<STR_LIT>" , args . hbase_config , host , job_name , instance_id ) <EOL> def show ( args ) : <EOL> get_hbase_service_config ( args ) <EOL> for job_name in args . job or ALL_JOBS : <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> task_list = deploy_utils . schedule_task_for_threads ( args , hosts , job_name , '<STR_LIT>' ) <EOL> parallel_deploy . start_deploy_threads ( show_job , task_list ) <EOL> def run_shell ( args ) : <EOL> get_hbase_service_config ( args ) <EOL> main_class , options = deploy_utils . parse_shell_command ( <EOL> args , SHELL_COMMAND_INFO ) <EOL> if not main_class : <EOL> return <EOL> args . hbase_config . parse_generated_config_files ( args ) <EOL> core_site_dict = args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] <EOL> hdfs_site_dict = args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] <EOL> hbase_site_dict = args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] <EOL> hbase_opts = list ( ) <EOL> for key , value in core_site_dict . iteritems ( ) : <EOL> hbase_opts . append ( "<STR_LIT>" % ( deploy_utils . HADOOP_PROPERTY_PREFIX , <EOL> key , value ) ) <EOL> for key , value in hdfs_site_dict . iteritems ( ) : <EOL> hbase_opts . append ( "<STR_LIT>" % ( deploy_utils . HADOOP_PROPERTY_PREFIX , <EOL> key , value ) ) <EOL> for key , value in hbase_site_dict . iteritems ( ) : <EOL> hbase_opts . append ( "<STR_LIT>" % ( deploy_utils . HADOOP_PROPERTY_PREFIX , <EOL> key , value ) ) <EOL> if deploy_utils . is_security_enabled ( args ) : <EOL> hbase_opts . append ( "<STR_LIT>" % <EOL> deploy_utils . get_config_dir ( ) ) <EOL> ( jaas_fd , jaas_file ) = tempfile . mkstemp ( ) <EOL> args . zookeeper_config . parse_generated_config_files ( args ) <EOL> os . write ( jaas_fd , deploy_zookeeper . generate_client_jaas_config ( args ) ) <EOL> os . close ( jaas_fd ) <EOL> hbase_opts . append ( "<STR_LIT>" % jaas_file ) <EOL> package_root = deploy_utils . get_artifact_package_root ( args , <EOL> args . hbase_config . cluster , "<STR_LIT>" ) <EOL> class_path = "<STR_LIT>" % ( package_root , package_root , package_root ) <EOL> cmd = [ "<STR_LIT>" , "<STR_LIT>" , class_path ] + hbase_opts + [ main_class ] <EOL> if args . command [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> cmd += [ "<STR_LIT>" , "<STR_LIT>" % package_root ] <EOL> cmd += options <EOL> p = subprocess . Popen ( cmd , stdout = sys . stdout , stderr = sys . stderr ) <EOL> return p . wait ( ) <EOL> def update_hbase_env_sh ( args , artifact , version ) : <EOL> current_path = os . path . abspath ( os . path . dirname ( <EOL> os . path . realpath ( args . package_root ) ) ) <EOL> conf_path = "<STR_LIT>" % ( current_path , args . package_root , <EOL> args . cluster , artifact , version ) <EOL> hbase_opts = "<STR_LIT>" <EOL> hbase_opts += "<STR_LIT>" <EOL> deploy_utils . append_to_file ( "<STR_LIT>" % conf_path , <EOL> '<STR_LIT>' % hbase_opts ) <EOL> def generate_client_config ( args , artifact , version ) : <EOL> config_path = "<STR_LIT>" % ( args . package_root , <EOL> args . cluster , artifact , version ) <EOL> master_host = args . hbase_config . jobs [ "<STR_LIT>" ] . hosts [ <NUM_LIT:0> ] . ip <EOL> config_path = "<STR_LIT>" % ( args . package_root , <EOL> args . cluster , artifact , version ) <EOL> deploy_utils . write_file ( "<STR_LIT>" % config_path , <EOL> deploy_utils . generate_site_xml ( args , <EOL> args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] ) ) <EOL> deploy_utils . write_file ( "<STR_LIT>" % config_path , <EOL> generate_metrics_config ( args , master_host , "<STR_LIT>" ) ) <EOL> deploy_utils . write_file ( "<STR_LIT>" % config_path , <EOL> deploy_utils . generate_site_xml ( args , <EOL> args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] ) ) <EOL> deploy_utils . write_file ( "<STR_LIT>" % config_path , <EOL> deploy_utils . generate_site_xml ( args , <EOL> args . hbase_config . configuration . generated_files [ "<STR_LIT>" ] ) ) <EOL> args . zookeeper_config . parse_generated_config_files ( args ) <EOL> deploy_utils . write_file ( "<STR_LIT>" % config_path , <EOL> deploy_zookeeper . generate_client_jaas_config ( args ) ) <EOL> deploy_utils . write_file ( "<STR_LIT>" % config_path , <EOL> args . hbase_config . configuration . raw_files [ "<STR_LIT>" ] ) <EOL> update_hbase_env_sh ( args , artifact , version ) <EOL> def pack ( args ) : <EOL> get_hbase_service_config ( args ) <EOL> args . hbase_config . parse_generated_config_files ( args ) <EOL> version = args . hbase_config . cluster . version <EOL> deploy_utils . make_package_dir ( args , "<STR_LIT>" , args . hbase_config . cluster ) <EOL> generate_client_config ( args , "<STR_LIT>" , version ) <EOL> if not args . skip_tarball : <EOL> deploy_utils . pack_package ( args , "<STR_LIT>" , version ) <EOL> Log . print_success ( "<STR_LIT>" ) <EOL> def vacate_region_server ( args , ip , port ) : <EOL> package_root = deploy_utils . get_artifact_package_root ( args , <EOL> args . hbase_config . cluster , "<STR_LIT>" ) <EOL> Log . print_info ( "<STR_LIT>" + ip ) ; <EOL> try : <EOL> host = socket . gethostbyaddr ( ip ) [ <NUM_LIT:0> ] <EOL> except : <EOL> host = ip <EOL> args . command = [ "<STR_LIT>" , "<STR_LIT>" % package_root , <EOL> "<STR_LIT>" , "<STR_LIT>" % ( host , port ) ] <EOL> if run_shell ( args ) != <NUM_LIT:0> : <EOL> Log . print_critical ( "<STR_LIT>" % host ) ; <EOL> def recover_region_server ( args , ip , port ) : <EOL> package_root = deploy_utils . get_artifact_package_root ( args , <EOL> args . hbase_config . cluster , "<STR_LIT>" ) <EOL> Log . print_info ( "<STR_LIT>" + ip ) ; <EOL> try : <EOL> host = socket . gethostbyaddr ( ip ) [ <NUM_LIT:0> ] <EOL> except : <EOL> host = ip <EOL> args . command = [ "<STR_LIT>" , "<STR_LIT>" % package_root , <EOL> "<STR_LIT>" , "<STR_LIT>" % ( host , port ) ] <EOL> if run_shell ( args ) != <NUM_LIT:0> : <EOL> Log . print_critical ( "<STR_LIT>" % host ) ; <EOL> def balance_switch ( args , flag ) : <EOL> fd , filename = tempfile . mkstemp ( ) <EOL> f = os . fdopen ( fd , '<STR_LIT>' ) <EOL> if flag : <EOL> Log . print_info ( "<STR_LIT>" % args . cluster ) <EOL> print >> f , '<STR_LIT>' <EOL> else : <EOL> Log . print_info ( "<STR_LIT>" % args . cluster ) <EOL> print >> f , '<STR_LIT>' <EOL> print >> f , '<STR_LIT>' <EOL> f . close ( ) <EOL> args . command = [ "<STR_LIT>" , filename ] <EOL> ret = run_shell ( args ) <EOL> os . remove ( filename ) <EOL> if ret != <NUM_LIT:0> : <EOL> Log . print_critical ( "<STR_LIT>" % <EOL> args . cluster ) ; <EOL> def rolling_update ( args ) : <EOL> if not args . job : <EOL> Log . print_critical ( "<STR_LIT>" ) <EOL> if not args . skip_confirm : <EOL> deploy_utils . confirm_action ( args , "<STR_LIT>" ) <EOL> get_hbase_service_config ( args ) <EOL> job_name = args . job [ <NUM_LIT:0> ] <EOL> if job_name != '<STR_LIT>' : <EOL> args . vacate_rs = False <EOL> if args . vacate_rs : <EOL> balance_switch ( args , False ) <EOL> Log . print_info ( "<STR_LIT>" % job_name ) <EOL> hosts = args . hbase_config . jobs [ job_name ] . hosts <EOL> wait_time = <NUM_LIT:0> <EOL> args . task_map = deploy_utils . parse_args_host_and_task ( args , hosts ) <EOL> for host_id in args . task_map . keys ( ) or hosts . iterkeys ( ) : <EOL> for instance_id in args . task_map . get ( host_id ) or range ( hosts [ host_id ] . instance_num ) : <EOL> instance_id = - <NUM_LIT:1> if not deploy_utils . is_multiple_instances ( host_id , hosts ) else instance_id <EOL> if not args . skip_confirm : <EOL> deploy_utils . confirm_rolling_update ( host_id , instance_id , wait_time ) <EOL> port = deploy_utils . get_base_port ( <EOL> args . hbase_config . jobs [ job_name ] . base_port , instance_id ) <EOL> if args . vacate_rs : <EOL> vacate_region_server ( args , hosts [ host_id ] . ip , port ) <EOL> stop_job ( args , hosts [ host_id ] . ip , job_name , instance_id ) <EOL> deploy_utils . wait_for_job_stopping ( "<STR_LIT>" , <EOL> args . hbase_config . cluster . name , job_name , hosts [ host_id ] . ip , instance_id ) <EOL> start_job ( args , hosts [ host_id ] . ip , job_name , host_id , instance_id ) <EOL> deploy_utils . wait_for_job_starting ( "<STR_LIT>" , <EOL> args . hbase_config . cluster . name , job_name , hosts [ host_id ] . ip , instance_id ) <EOL> if args . vacate_rs : <EOL> recover_region_server ( args , hosts [ host_id ] . ip , port ) <EOL> wait_time = args . time_interval <EOL> if args . vacate_rs : <EOL> balance_switch ( args , True ) <EOL> Log . print_success ( "<STR_LIT>" % job_name ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> test ( ) </s>
