<s> from StringIO import StringIO <EOL> import time <EOL> from fabric import api as fab <EOL> from fabric . tasks import execute <EOL> import os <EOL> import yaml <EOL> from cluster_config import config as cluster_config <EOL> import re <EOL> import sh <EOL> import uuid <EOL> from util import random_token , get_static_vnode_tokens <EOL> import subprocess <EOL> from util import random_token <EOL> import fab_dse as dse <EOL> import fab_cassandra as cstar <EOL> import fab_flamegraph as flamegraph <EOL> import fab_profiler as profiler <EOL> import logging <EOL> import pkg_resources <EOL> logging . basicConfig ( ) <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> logger . setLevel ( logging . INFO ) <EOL> fab . env . use_ssh_config = True <EOL> fab . env . connection_attempts = <NUM_LIT:10> <EOL> git_repos = [ <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ] <EOL> CMD_LINE_HOSTS_SPECIFIED = False <EOL> if len ( fab . env . hosts ) > <NUM_LIT:0> : <EOL> CMD_LINE_HOSTS_SPECIFIED = True <EOL> logback_template = """<STR_LIT>""" <EOL> log4j_template = """<STR_LIT>""" <EOL> DENIED_CSTAR_CONFIG = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> def setup ( my_config = None ) : <EOL> global config <EOL> def __get_balanced_tokens ( node_count , partitioner = '<STR_LIT>' ) : <EOL> if partitioner == '<STR_LIT>' : <EOL> return [ str ( ( ( <NUM_LIT:2> ** <NUM_LIT:64> / node_count ) * i ) - <NUM_LIT:2> ** <NUM_LIT> ) <EOL> for i in range ( node_count ) ] <EOL> elif partitioner == '<STR_LIT>' : <EOL> return [ str ( i * ( <NUM_LIT:2> ** <NUM_LIT> / node_count ) ) for i in range ( <NUM_LIT:0> , node_count ) ] <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' % partitioner ) <EOL> default_config = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( random_string = random_token ( ) ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:user>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : False <EOL> } <EOL> public_ips = "<STR_LIT>" <EOL> private_ips = "<STR_LIT>" <EOL> public_ips = public_ips . replace ( "<STR_LIT:U+0020>" , "<STR_LIT>" ) . split ( "<STR_LIT:U+002C>" ) <EOL> private_ips = private_ips . replace ( "<STR_LIT:U+0020>" , "<STR_LIT>" ) . split ( "<STR_LIT:U+002C>" ) <EOL> tokens = __get_balanced_tokens ( len ( public_ips ) , default_config [ '<STR_LIT>' ] ) <EOL> default_config . setdefault ( '<STR_LIT>' , { } ) <EOL> first_node = True <EOL> for i , public , private , token in zip ( <EOL> xrange ( len ( public_ips ) ) , public_ips , private_ips , tokens ) : <EOL> if not default_config [ '<STR_LIT>' ] . has_key ( public ) : <EOL> default_config [ '<STR_LIT>' ] [ public ] = { <EOL> '<STR_LIT>' : '<STR_LIT>' % i , <EOL> '<STR_LIT>' : private , <EOL> } <EOL> if not default_config [ '<STR_LIT>' ] : <EOL> default_config [ '<STR_LIT>' ] [ public ] [ '<STR_LIT>' ] = token <EOL> if first_node : <EOL> default_config [ '<STR_LIT>' ] [ public ] [ '<STR_LIT>' ] = True <EOL> first_node = False <EOL> if not my_config : <EOL> config = default_config <EOL> else : <EOL> config = dict ( default_config . items ( ) + my_config . items ( ) ) <EOL> for node in config [ '<STR_LIT>' ] . values ( ) : <EOL> if not config [ '<STR_LIT>' ] and not node . has_key ( '<STR_LIT>' ) : <EOL> tokens = __get_balanced_tokens ( len ( config [ '<STR_LIT>' ] ) , config [ '<STR_LIT>' ] ) <EOL> for node , token in zip ( config [ '<STR_LIT>' ] . values ( ) , tokens ) : <EOL> node [ '<STR_LIT>' ] = token <EOL> break <EOL> config [ '<STR_LIT>' ] = [ v . get ( '<STR_LIT>' , v [ '<STR_LIT>' ] ) for v in config [ '<STR_LIT>' ] . values ( ) <EOL> if v . get ( '<STR_LIT>' , False ) ] <EOL> if not CMD_LINE_HOSTS_SPECIFIED : <EOL> fab . env . hosts = [ h for h in config [ '<STR_LIT>' ] ] <EOL> fab . env . user = config [ '<STR_LIT:user>' ] <EOL> setup ( ) <EOL> setup ( cluster_config ) <EOL> @ fab . parallel <EOL> def bootstrap ( git_fetch = True , revision_override = None ) : <EOL> """<STR_LIT>""" <EOL> partitioner = config [ '<STR_LIT>' ] <EOL> fab . run ( '<STR_LIT>' ) <EOL> if config [ '<STR_LIT>' ] in ( None , '<STR_LIT>' ) : <EOL> logger . warn ( "<STR_LIT>" ) <EOL> config [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> elif config [ '<STR_LIT>' ] not in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> product = dse if config [ '<STR_LIT>' ] == '<STR_LIT>' else cstar <EOL> if product . name == '<STR_LIT>' : <EOL> rev_id = dse . bootstrap ( config ) <EOL> else : <EOL> rev_id = cstar . bootstrap ( config , git_fetch = git_fetch , revision_override = revision_override ) <EOL> cassandra_path = product . get_cassandra_path ( ) <EOL> try : <EOL> cfg = config [ '<STR_LIT>' ] [ fab . env . host ] <EOL> except KeyError : <EOL> return rev_id <EOL> if config [ '<STR_LIT>' ] : <EOL> jna_jars = os . path . join ( cassandra_path , '<STR_LIT>' ) <EOL> jna_jar = os . path . join ( cassandra_path , '<STR_LIT>' ) <EOL> jna_exists = fab . run ( '<STR_LIT>' . format ( jna_jars ) , quiet = True ) <EOL> if jna_exists . return_code != <NUM_LIT:0> : <EOL> jna_candidates = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> for jar in jna_candidates : <EOL> if fab . run ( '<STR_LIT>' . format ( jar = jar ) , quiet = True ) . return_code == <NUM_LIT:0> : <EOL> fab . run ( '<STR_LIT>' . format ( jar = jar , jna = jna_jar ) ) <EOL> break <EOL> else : <EOL> if not os . path . exists ( '<STR_LIT>' ) : <EOL> request = download_file ( JNA_LIB_URL , '<STR_LIT>' ) <EOL> if request . status_code != requests . codes . ok : <EOL> raise AssertionError ( '<STR_LIT>' ) <EOL> fab . put ( '<STR_LIT>' , jna_jar ) <EOL> else : <EOL> fab . run ( '<STR_LIT>' . format ( os . path . join ( cassandra_path , '<STR_LIT>' ) ) ) <EOL> conf_file = StringIO ( ) <EOL> fab . get ( os . path . join ( cassandra_path , '<STR_LIT>' ) , conf_file ) <EOL> conf_file . seek ( <NUM_LIT:0> ) <EOL> cass_yaml = yaml . load ( conf_file . read ( ) ) <EOL> cstar_config_opts = product . get_cassandra_config_options ( config ) <EOL> try : <EOL> cstar_config_opts . remove ( '<STR_LIT>' ) <EOL> except ValueError : <EOL> pass <EOL> if product . name == '<STR_LIT>' : <EOL> dse_config_options = product . get_dse_config_options ( config ) <EOL> dse_conf_file = StringIO ( ) <EOL> dse_yaml_path = os . path . join ( product . get_dse_conf_path ( ) , '<STR_LIT>' ) <EOL> fab . get ( dse_yaml_path , dse_conf_file ) <EOL> dse_conf_file . seek ( <NUM_LIT:0> ) <EOL> dse_yaml = yaml . load ( dse_conf_file . read ( ) ) <EOL> configured_dse_yaml_settings = config . get ( '<STR_LIT>' , { } ) <EOL> if configured_dse_yaml_settings : <EOL> for option , value in configured_dse_yaml_settings . items ( ) : <EOL> if option not in dse_config_options : <EOL> raise ValueError ( '<STR_LIT>' . format ( option ) ) <EOL> dse_yaml [ option ] = value <EOL> dse_conf_file = StringIO ( ) <EOL> dse_conf_file . write ( yaml . safe_dump ( dse_yaml , encoding = '<STR_LIT:utf-8>' , allow_unicode = True ) ) <EOL> dse_conf_file . seek ( <NUM_LIT:0> ) <EOL> fab . put ( dse_conf_file , dse_yaml_path ) <EOL> for option , value in config . items ( ) : <EOL> if option in cstar_config_opts : <EOL> cass_yaml [ option ] = value <EOL> for option , value in config . get ( '<STR_LIT>' , { } ) . items ( ) : <EOL> if option in DENIED_CSTAR_CONFIG : <EOL> raise ValueError ( <EOL> '<STR_LIT>' . format ( option ) <EOL> ) <EOL> elif option not in cstar_config_opts : <EOL> raise ValueError ( '<STR_LIT>' . format ( option ) ) <EOL> cass_yaml [ option ] = value <EOL> if '<STR_LIT>' not in config . get ( '<STR_LIT>' , { } ) : <EOL> if config . get ( '<STR_LIT>' , True ) : <EOL> cass_yaml [ '<STR_LIT>' ] = config [ '<STR_LIT>' ] <EOL> else : <EOL> cass_yaml [ '<STR_LIT>' ] = cfg [ '<STR_LIT>' ] <EOL> cass_yaml [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> cass_yaml [ '<STR_LIT>' ] = cfg [ '<STR_LIT>' ] <EOL> cass_yaml [ '<STR_LIT>' ] = cfg . get ( '<STR_LIT>' , cfg [ '<STR_LIT>' ] ) <EOL> cass_yaml [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] = "<STR_LIT:U+002C>" . join ( config [ '<STR_LIT>' ] ) <EOL> if partitioner == '<STR_LIT>' : <EOL> cass_yaml [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> elif partitioner == '<STR_LIT>' : <EOL> cass_yaml [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> cass_yaml [ '<STR_LIT>' ] = cfg [ '<STR_LIT>' ] <EOL> if not config . has_key ( '<STR_LIT>' ) : <EOL> for node in config [ '<STR_LIT>' ] . values ( ) : <EOL> if node . get ( '<STR_LIT>' , False ) : <EOL> config [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> cass_yaml [ '<STR_LIT>' ] = False <EOL> break <EOL> else : <EOL> config [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> conf_dir = os . path . join ( cassandra_path , '<STR_LIT>' ) <EOL> if config [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> cass_yaml [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> fab . run ( "<STR_LIT>" . format ( conf_dir + '<STR_LIT>' ) ) <EOL> for node in config [ '<STR_LIT>' ] . values ( ) : <EOL> line = '<STR_LIT>' % ( node [ '<STR_LIT>' ] , node . get ( '<STR_LIT>' , '<STR_LIT>' ) , node . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> fab . run ( "<STR_LIT>" . format ( line , conf_dir + '<STR_LIT>' ) ) <EOL> if config [ '<STR_LIT>' ] == "<STR_LIT>" : <EOL> cass_yaml [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> fab . run ( "<STR_LIT>" . format ( <EOL> dc = cfg . get ( '<STR_LIT>' , '<STR_LIT>' ) , rack = cfg . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> out = conf_dir + '<STR_LIT>' ) ) <EOL> conf_file = StringIO ( ) <EOL> conf_file . write ( yaml . safe_dump ( cass_yaml , encoding = '<STR_LIT:utf-8>' , allow_unicode = True ) ) <EOL> conf_file . seek ( <NUM_LIT:0> ) <EOL> fab . put ( conf_file , conf_dir + '<STR_LIT>' ) <EOL> logback_conf = StringIO ( ) <EOL> log_dir = fab . run ( "<STR_LIT>" . format ( log_dir = config [ '<STR_LIT>' ] ) ) <EOL> logback_conf . write ( logback_template . replace ( "<STR_LIT>" , log_dir ) ) <EOL> logback_conf . seek ( <NUM_LIT:0> ) <EOL> fab . put ( logback_conf , conf_dir + '<STR_LIT>' ) <EOL> log4j_conf = StringIO ( ) <EOL> log4j_conf . write ( log4j_template . replace ( "<STR_LIT>" , log_dir ) ) <EOL> log4j_conf . seek ( <NUM_LIT:0> ) <EOL> fab . put ( log4j_conf , conf_dir + '<STR_LIT>' ) <EOL> fincore_script = os . path . join ( os . path . dirname ( os . path . realpath ( __file__ ) ) , '<STR_LIT>' ) <EOL> fab . put ( fincore_script , '<STR_LIT>' ) <EOL> return rev_id <EOL> @ fab . parallel <EOL> def destroy ( leave_data = False , kill_delay = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> if leave_data : <EOL> fab . run ( '<STR_LIT>' . format ( java_home = config [ '<STR_LIT>' ] , nodetool_cmd = _nodetool_cmd ( ) ) , quiet = True ) <EOL> fab . run ( '<STR_LIT>' . format ( commitlog = config [ '<STR_LIT>' ] ) ) <EOL> if kill_delay : <EOL> fab . run ( '<STR_LIT>' , quiet = True ) <EOL> time . sleep ( kill_delay ) <EOL> fab . run ( '<STR_LIT>' , quiet = True ) <EOL> fab . run ( '<STR_LIT>' , quiet = True ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> assert type ( config [ '<STR_LIT>' ] ) == list <EOL> for t in [ config [ '<STR_LIT>' ] , config [ '<STR_LIT>' ] , <EOL> config [ '<STR_LIT>' ] , config [ '<STR_LIT>' ] ] + config [ '<STR_LIT>' ] : <EOL> assert type ( t ) in ( str , unicode ) and len ( t ) > <NUM_LIT:1> , '<STR_LIT>' . format ( t = t ) <EOL> if not leave_data : <EOL> for d in config [ '<STR_LIT>' ] : <EOL> fab . run ( '<STR_LIT>' . format ( data = d ) ) <EOL> fab . run ( '<STR_LIT>' . format ( saved_caches_directory = config [ '<STR_LIT>' ] ) ) <EOL> fab . run ( '<STR_LIT>' . format ( commitlog = config [ '<STR_LIT>' ] ) ) <EOL> fab . run ( '<STR_LIT>' . format ( flushdir = config [ '<STR_LIT>' ] ) ) <EOL> if config . get ( '<STR_LIT>' ) : <EOL> fab . run ( '<STR_LIT>' . format ( hints_directory = config . get ( '<STR_LIT>' ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( log_dir = config [ '<STR_LIT>' ] ) ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> @ fab . parallel <EOL> def start ( ) : <EOL> """<STR_LIT>""" <EOL> product = dse if config [ '<STR_LIT>' ] == '<STR_LIT>' else cstar <EOL> cassandra_path = product . get_cassandra_path ( ) <EOL> env = config . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> fab . puts ( '<STR_LIT>' . format ( env ) ) <EOL> if isinstance ( env , list ) or isinstance ( env , tuple ) : <EOL> env = "<STR_LIT:\n>" . join ( env ) <EOL> env += "<STR_LIT:\n>" <EOL> fab . puts ( '<STR_LIT>' . format ( env ) ) <EOL> if not config [ '<STR_LIT>' ] : <EOL> env = '<STR_LIT>' + env <EOL> fab . run ( "<STR_LIT>" ) <EOL> log_dir = fab . run ( "<STR_LIT>" . format ( log_dir = config [ '<STR_LIT>' ] ) ) <EOL> try : <EOL> ip_address = cluster_config [ '<STR_LIT>' ] [ fab . env . host ] [ '<STR_LIT>' ] <EOL> except : <EOL> ip_address = fab . env . host <EOL> env = "<STR_LIT>" . format ( <EOL> hostname = ip_address , log_dir = log_dir ) + env <EOL> env = "<STR_LIT>" + env <EOL> if flamegraph . is_enabled ( config ) : <EOL> env += "<STR_LIT>" <EOL> if profiler . yourkit_is_enabled ( config ) : <EOL> execute ( profiler . yourkit_clean ) <EOL> env += profiler . yourkit_get_jvm_opts ( ) <EOL> fab . puts ( "<STR_LIT>" . format ( config [ '<STR_LIT>' ] ) ) <EOL> if config [ '<STR_LIT>' ] and config [ '<STR_LIT>' ] in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> env += "<STR_LIT>" . format ( <EOL> get_static_vnode_tokens ( fab . env . host , <EOL> fab . env . hosts , <EOL> partitioner = config [ '<STR_LIT>' ] , <EOL> group = config [ '<STR_LIT>' ] ) ) <EOL> env_script = "<STR_LIT>" . format ( name = uuid . uuid1 ( ) ) <EOL> env_file = StringIO ( env ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> fab . put ( env_file , '<STR_LIT>' . format ( env_script = env_script ) ) <EOL> fab . puts ( '<STR_LIT>' . format ( env ) ) <EOL> if len ( env_script ) > <NUM_LIT:0> : <EOL> env_path = os . path . join ( cassandra_path , '<STR_LIT>' ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> product . start ( config ) <EOL> @ fab . parallel <EOL> def stop ( clean = True ) : <EOL> product = dse if config [ '<STR_LIT>' ] == '<STR_LIT>' else cstar <EOL> product . stop ( clean , config ) <EOL> def _nodetool_cmd ( ) : <EOL> product = dse if config [ '<STR_LIT>' ] == '<STR_LIT>' else cstar <EOL> bin_path = product . get_bin_path ( ) <EOL> return os . path . join ( bin_path , '<STR_LIT>' ) <EOL> @ fab . parallel <EOL> def multi_nodetool ( cmd ) : <EOL> """<STR_LIT>""" <EOL> return fab . run ( '<STR_LIT>' . format ( java_home = config [ '<STR_LIT>' ] , nodetool_cmd = _nodetool_cmd ( ) , cmd = cmd ) , warn_only = True ) <EOL> def ensure_running ( retries = <NUM_LIT:15> , wait = <NUM_LIT:10> ) : <EOL> """<STR_LIT>""" <EOL> time . sleep ( <NUM_LIT:15> ) <EOL> for attempt in range ( retries ) : <EOL> ring = StringIO ( fab . run ( '<STR_LIT>' . format ( <EOL> java_home = config [ '<STR_LIT>' ] , nodetool_bin = _nodetool_cmd ( ) ) ) ) <EOL> broadcast_ips = [ x . get ( '<STR_LIT>' , x [ '<STR_LIT>' ] ) for x in config [ '<STR_LIT>' ] . values ( ) ] <EOL> nodes_up = dict ( ( host , False ) for host in broadcast_ips ) <EOL> for line in ring : <EOL> for host in broadcast_ips : <EOL> try : <EOL> if host in line and "<STR_LIT>" in line : <EOL> nodes_up [ host ] = True <EOL> except UnicodeDecodeError : <EOL> pass <EOL> for node , up in nodes_up . items ( ) : <EOL> if not up : <EOL> fab . puts ( "<STR_LIT>" % node ) <EOL> if False not in nodes_up . values ( ) : <EOL> fab . puts ( "<STR_LIT>" ) <EOL> return <EOL> fab . puts ( "<STR_LIT>" % wait ) <EOL> time . sleep ( wait ) <EOL> else : <EOL> fab . abort ( "<STR_LIT>" ) <EOL> @ fab . parallel <EOL> def ensure_stopped ( retries = <NUM_LIT:15> , wait = <NUM_LIT:10> ) : <EOL> """<STR_LIT>""" <EOL> product = dse if config [ '<STR_LIT>' ] == '<STR_LIT>' else cstar <EOL> for attempt in range ( retries ) : <EOL> pgrep = fab . run ( '<STR_LIT>' , quiet = True ) <EOL> if not product . is_running ( ) : <EOL> fab . puts ( '<STR_LIT>' ) <EOL> return <EOL> fab . puts ( "<STR_LIT>" % wait ) <EOL> time . sleep ( wait ) <EOL> else : <EOL> fab . abort ( "<STR_LIT>" ) <EOL> @ fab . parallel <EOL> def install_java ( packages = None ) : <EOL> dist = fab . run ( '<STR_LIT>' , quiet = True ) <EOL> if dist . return_code != <NUM_LIT:0> : <EOL> dist = fab . run ( '<STR_LIT>' , quiet = True ) <EOL> if dist . startswith ( '<STR_LIT>' ) : <EOL> if not packages : <EOL> packages = [ '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> cmd = '<STR_LIT>' <EOL> elif dist . startswith ( '<STR_LIT>' ) : <EOL> if not packages : <EOL> packages = [ '<STR_LIT>' ] <EOL> fab . run ( '<STR_LIT>' ) <EOL> cmd = '<STR_LIT>' <EOL> else : <EOL> raise RuntimeError ( '<STR_LIT>' % dist ) <EOL> for package in packages : <EOL> fab . run ( cmd . format ( package = package ) ) <EOL> @ fab . parallel <EOL> def configure_hostnames ( ) : <EOL> cfg = config [ '<STR_LIT>' ] [ fab . env . host ] <EOL> fab . run ( '<STR_LIT>' . format ( cfg [ '<STR_LIT>' ] ) ) <EOL> fab . run ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> for cfg in config [ '<STR_LIT>' ] . values ( ) : <EOL> fab . run ( '<STR_LIT>' . format ( <EOL> ip = cfg [ '<STR_LIT>' ] , hostname = cfg [ '<STR_LIT>' ] ) ) <EOL> @ fab . parallel <EOL> def copy_logs ( local_directory ) : <EOL> cfg = config [ '<STR_LIT>' ] [ fab . env . host ] <EOL> host_log_dir = os . path . join ( local_directory , cfg [ '<STR_LIT>' ] ) <EOL> os . mkdir ( host_log_dir ) <EOL> fab . get ( os . path . join ( config [ '<STR_LIT>' ] , '<STR_LIT:*>' ) , host_log_dir ) <EOL> @ fab . parallel <EOL> def start_fincore_capture ( interval = <NUM_LIT:10> ) : <EOL> """<STR_LIT>""" <EOL> fab . puts ( "<STR_LIT>" ) <EOL> fab . run ( '<STR_LIT>' . format ( interval = interval ) ) <EOL> @ fab . parallel <EOL> def stop_fincore_capture ( ) : <EOL> """<STR_LIT>""" <EOL> fab . puts ( "<STR_LIT>" ) <EOL> fab . run ( '<STR_LIT>' , quiet = True ) <EOL> @ fab . parallel <EOL> def copy_fincore_logs ( local_directory ) : <EOL> cfg = config [ '<STR_LIT>' ] [ fab . env . host ] <EOL> location = os . path . join ( local_directory , "<STR_LIT>" . format ( host = cfg [ '<STR_LIT>' ] ) ) <EOL> fab . get ( '<STR_LIT>' , location ) <EOL> @ fab . parallel <EOL> def whoami ( ) : <EOL> fab . run ( '<STR_LIT>' ) <EOL> @ fab . parallel <EOL> def copy_root_setup ( ) : <EOL> fab . run ( '<STR_LIT>' ) <EOL> @ fab . parallel <EOL> def set_device_read_ahead ( read_ahead , devices ) : <EOL> with fab . settings ( user = '<STR_LIT:root>' ) : <EOL> for device in devices : <EOL> if '<STR_LIT>' in device : <EOL> continue <EOL> fab . run ( '<STR_LIT>' . format ( read_ahead = read_ahead , device = device ) ) <EOL> @ fab . parallel <EOL> def build_jbod_drives ( device_mounts , md_device = '<STR_LIT>' , filesystem = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> with fab . settings ( user = '<STR_LIT:root>' ) : <EOL> if isinstance ( device_mounts , basestring ) : <EOL> device_mounts = eval ( device_mounts ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) , quiet = True ) <EOL> mounted = fab . run ( '<STR_LIT>' , quiet = True ) <EOL> for device in [ d for d in device_mounts . keys ( ) if d in mounted ] : <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( devices = "<STR_LIT:\n>" . join ( device_mounts . keys ( ) ) , filesystem = filesystem , num = len ( device_mounts ) ) ) <EOL> for device , mountpoint in device_mounts . items ( ) : <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> @ fab . parallel <EOL> def build_striped_drives ( devices , mount_point = '<STR_LIT>' , filesystem = '<STR_LIT>' , chunk_size = <NUM_LIT:64> , md_device = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> with fab . settings ( user = '<STR_LIT:root>' ) : <EOL> if isinstance ( devices , basestring ) : <EOL> devices = devices . split ( "<STR_LIT:;>" ) <EOL> mounted = fab . run ( '<STR_LIT>' , quiet = True ) <EOL> for device in [ d for d in devices + [ mount_point ] if d in mounted ] : <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) , quiet = True ) <EOL> fab . run ( '<STR_LIT>' . format ( <EOL> md_device = md_device , num = len ( devices ) , chunk_size = chunk_size , devices = "<STR_LIT:U+0020>" . join ( devices ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> fab . run ( '<STR_LIT>' . format ( ** locals ( ) ) ) <EOL> def parse_output ( output ) : <EOL> """<STR_LIT>""" <EOL> line_pattern = re . compile ( '<STR_LIT>' ) <EOL> results = { } <EOL> for line in output : <EOL> m = line_pattern . match ( line ) <EOL> if m : <EOL> host = m . group ( '<STR_LIT:host>' ) <EOL> if host not in results : <EOL> results [ host ] = [ ] <EOL> line = m . group ( '<STR_LIT>' ) . strip ( ) <EOL> if line : <EOL> results [ host ] . append ( line ) <EOL> return results <EOL> @ fab . parallel <EOL> def bash ( script ) : <EOL> """<STR_LIT>""" <EOL> script = StringIO ( script ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> script_path = '<STR_LIT>' . format ( script_name = uuid . uuid1 ( ) ) <EOL> fab . put ( script , script_path ) <EOL> output = StringIO ( ) <EOL> fab . run ( '<STR_LIT>' . format ( script_path = script_path ) , stdout = output , stderr = output ) <EOL> output . seek ( <NUM_LIT:0> ) <EOL> return output . read ( ) . splitlines ( ) <EOL> def runbg ( cmd , envs , sockname = "<STR_LIT>" ) : <EOL> env_vars = "<STR_LIT>" <EOL> for var , value in envs . iteritems ( ) : <EOL> env_vars += "<STR_LIT>" . format ( var , value ) <EOL> cmd_ = '<STR_LIT>' . format ( env_vars , sockname , cmd ) <EOL> logger . info ( "<STR_LIT>" . format ( cmd_ ) ) <EOL> return fab . run ( cmd_ ) <EOL> @ fab . parallel <EOL> def python ( script ) : <EOL> """<STR_LIT>""" <EOL> script = StringIO ( script ) <EOL> fab . run ( '<STR_LIT>' ) <EOL> script_path = '<STR_LIT>' . format ( script_name = uuid . uuid1 ( ) ) <EOL> fab . put ( script , script_path ) <EOL> output = StringIO ( ) <EOL> with fab . settings ( warn_only = True ) : <EOL> retval = fab . run ( <EOL> '<STR_LIT>' . format ( script_path = script_path ) , <EOL> stdout = output , stderr = output <EOL> ) <EOL> output . seek ( <NUM_LIT:0> ) <EOL> output = output . read ( ) <EOL> if retval . return_code != <NUM_LIT:0> : <EOL> logger . info ( output ) <EOL> raise Exception ( '<STR_LIT>' ) <EOL> return output . splitlines ( ) <EOL> def run_python_script ( script_name , function_name , parameters ) : <EOL> """<STR_LIT>""" <EOL> logger . info ( '<STR_LIT>' . format ( <EOL> script_name , <EOL> function_name , <EOL> parameters <EOL> ) ) <EOL> resource_package = __name__ <EOL> resource_path = os . path . join ( '<STR_LIT>' , '<STR_LIT>' . format ( script_name ) ) <EOL> script = pkg_resources . resource_string ( resource_package , resource_path ) <EOL> script = script . replace ( '<STR_LIT>' , function_name ) . replace ( '<STR_LIT>' , parameters ) <EOL> return python ( script ) </s>
