<s> """<STR_LIT>""" <EOL> import os , sys , yaml , urllib2 , logging , hashlib , time , subprocess , random <EOL> from urlparse import urlparse <EOL> from boto . s3 . key import Key <EOL> from boto . s3 . connection import S3Connection <EOL> from boto . exception import S3ResponseError <EOL> from boto . s3 . connection import OrdinaryCallingFormat <EOL> logging . getLogger ( '<STR_LIT>' ) . setLevel ( logging . INFO ) <EOL> log = None <EOL> USER_DATA_URL = '<STR_LIT>' <EOL> LOCAL_PATH = '<STR_LIT>' <EOL> USER_DATA_FILE_NAME = '<STR_LIT>' <EOL> USER_DATA_FILE = os . path . join ( LOCAL_PATH , USER_DATA_FILE_NAME ) <EOL> USER_DATA_ORIG = os . path . join ( LOCAL_PATH , '<STR_LIT>' % USER_DATA_FILE_NAME ) <EOL> SERVICE_ROOT = '<STR_LIT>' <EOL> DEFAULT_BUCKET_NAME = '<STR_LIT>' <EOL> DEFAULT_BOOT_SCRIPT_NAME = '<STR_LIT>' <EOL> CLOUDMAN_HOME = '<STR_LIT>' <EOL> def _setup_logging ( ) : <EOL> formatter = logging . Formatter ( "<STR_LIT>" ) <EOL> console = logging . StreamHandler ( ) <EOL> console . setFormatter ( formatter ) <EOL> log = logging . root <EOL> log . addHandler ( console ) <EOL> log . setLevel ( logging . DEBUG ) <EOL> return log <EOL> def _get_user_data ( ) : <EOL> ud = '<STR_LIT>' <EOL> for i in range ( <NUM_LIT:0> , <NUM_LIT:5> ) : <EOL> try : <EOL> log . info ( "<STR_LIT>" % ( USER_DATA_URL , i ) ) <EOL> fp = urllib2 . urlopen ( USER_DATA_URL ) <EOL> ud = fp . read ( ) <EOL> fp . close ( ) <EOL> log . debug ( "<STR_LIT>" % USER_DATA_ORIG ) <EOL> with open ( USER_DATA_ORIG , '<STR_LIT:w>' ) as ud_orig : <EOL> ud_orig . write ( ud ) <EOL> if ud : <EOL> log . debug ( "<STR_LIT>" ) <EOL> return ud <EOL> except IOError : <EOL> log . info ( "<STR_LIT>" ) <EOL> return '<STR_LIT>' <EOL> if ud == '<STR_LIT>' : <EOL> log . debug ( "<STR_LIT>" ) <EOL> return ud <EOL> def _get_bucket_name ( cluster_name , access_key ) : <EOL> """<STR_LIT>""" <EOL> m = hashlib . md5 ( ) <EOL> m . update ( cluster_name + access_key ) <EOL> return "<STR_LIT>" + m . hexdigest ( ) <EOL> def _isurl ( path ) : <EOL> """<STR_LIT>""" <EOL> scheme , netloc , upath , uparams , uquery , ufrag = urlparse ( path ) <EOL> return bool ( scheme and netloc ) <EOL> def _get_s3_conn ( ud ) : <EOL> try : <EOL> if '<STR_LIT>' in ud and ud [ '<STR_LIT>' ] != '<STR_LIT>' : <EOL> log . debug ( '<STR_LIT>' ) <EOL> try : <EOL> s3_conn = S3Connection ( aws_access_key_id = ud [ '<STR_LIT>' ] , <EOL> aws_secret_access_key = ud [ '<STR_LIT>' ] , <EOL> is_secure = ud . get ( '<STR_LIT>' , True ) , <EOL> host = ud . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> port = ud . get ( '<STR_LIT>' , <NUM_LIT> ) , <EOL> calling_format = OrdinaryCallingFormat ( ) , <EOL> path = ud . get ( '<STR_LIT>' , '<STR_LIT:/>' ) ) <EOL> except S3ResponseError , e : <EOL> log . error ( "<STR_LIT>" . format ( ud , e ) ) <EOL> else : <EOL> log . debug ( '<STR_LIT>' ) <EOL> s3_conn = S3Connection ( ud [ '<STR_LIT>' ] , ud [ '<STR_LIT>' ] ) <EOL> except Exception , e : <EOL> log . error ( "<STR_LIT>" % e ) <EOL> return None <EOL> return s3_conn <EOL> def _bucket_exists ( s3_conn , bucket_name ) : <EOL> bucket = None <EOL> for i in range ( <NUM_LIT:1> , <NUM_LIT:6> ) : <EOL> try : <EOL> bucket = s3_conn . lookup ( bucket_name ) <EOL> break <EOL> except S3ResponseError : <EOL> log . error ( "<STR_LIT>" % ( bucket_name , i + <NUM_LIT:1> ) ) <EOL> time . sleep ( <NUM_LIT:2> ) <EOL> if bucket is not None : <EOL> log . debug ( "<STR_LIT>" % bucket_name ) <EOL> return True <EOL> else : <EOL> log . debug ( "<STR_LIT>" % bucket_name ) <EOL> return False <EOL> def _remote_file_exists ( s3_conn , bucket_name , remote_filename ) : <EOL> b = None <EOL> for i in range ( <NUM_LIT:0> , <NUM_LIT:5> ) : <EOL> try : <EOL> b = s3_conn . get_bucket ( bucket_name ) <EOL> break <EOL> except S3ResponseError : <EOL> log . error ( "<STR_LIT>" % ( bucket_name , i ) ) <EOL> time . sleep ( <NUM_LIT:2> ) <EOL> if b is not None : <EOL> k = Key ( b , remote_filename ) <EOL> if k . exists ( ) : <EOL> return True <EOL> return False <EOL> def _save_file_to_bucket ( s3_conn , bucket_name , remote_filename , local_file , force = False ) : <EOL> local_file = os . path . join ( LOCAL_PATH , local_file ) <EOL> b = None <EOL> for i in range ( <NUM_LIT:0> , <NUM_LIT:5> ) : <EOL> try : <EOL> b = s3_conn . get_bucket ( bucket_name ) <EOL> break <EOL> except S3ResponseError , e : <EOL> log . error ( "<STR_LIT>" % ( bucket_name , i ) ) <EOL> time . sleep ( <NUM_LIT:2> ) <EOL> if b is not None : <EOL> k = Key ( b , remote_filename ) <EOL> if k . exists ( ) and not force : <EOL> log . debug ( "<STR_LIT>" % remote_filename ) <EOL> return True <EOL> log . debug ( "<STR_LIT>" <EOL> % ( local_file , bucket_name , remote_filename ) ) <EOL> try : <EOL> k . set_contents_from_filename ( local_file ) <EOL> log . info ( "<STR_LIT>" % ( remote_filename , bucket_name ) ) <EOL> return True <EOL> except S3ResponseError , e : <EOL> log . error ( "<STR_LIT>" <EOL> % ( local_file , bucket_name , remote_filename , e ) ) <EOL> return False <EOL> else : <EOL> return False <EOL> def _get_file_from_bucket ( s3_conn , bucket_name , remote_filename , local_filename ) : <EOL> local_filename = os . path . join ( LOCAL_PATH , local_filename ) <EOL> try : <EOL> b = s3_conn . get_bucket ( bucket_name ) <EOL> k = Key ( b , remote_filename ) <EOL> log . debug ( "<STR_LIT>" % ( remote_filename , bucket_name ) ) <EOL> if k . exists ( ) : <EOL> k . get_contents_to_filename ( local_filename ) <EOL> log . info ( "<STR_LIT>" <EOL> % ( remote_filename , bucket_name , local_filename ) ) <EOL> return True <EOL> else : <EOL> log . error ( "<STR_LIT>" % ( remote_filename , bucket_name ) ) <EOL> return False <EOL> except S3ResponseError , e : <EOL> log . error ( "<STR_LIT>" % ( remote_filename , bucket_name , e ) ) <EOL> return False <EOL> def _get_file_from_url ( url ) : <EOL> local_filename = os . path . join ( LOCAL_PATH , os . path . split ( url ) [ <NUM_LIT:1> ] ) <EOL> log . info ( "<STR_LIT>" % ( url , local_filename ) ) <EOL> try : <EOL> f = urllib2 . urlopen ( url ) <EOL> with open ( local_filename , '<STR_LIT:w>' ) as local_file : <EOL> local_file . write ( f . read ( ) ) <EOL> os . chmod ( local_filename , <NUM_LIT:0> <NUM_LIT> ) <EOL> if f : <EOL> log . debug ( "<STR_LIT>" % url ) <EOL> return True <EOL> return False <EOL> except IOError : <EOL> log . error ( "<STR_LIT>" % url ) <EOL> return False <EOL> def _get_boot_script ( ud ) : <EOL> use_default_bucket = ud . get ( "<STR_LIT>" , False ) <EOL> if ud . has_key ( '<STR_LIT>' ) : <EOL> default_bucket_name = ud [ '<STR_LIT>' ] <EOL> else : <EOL> default_bucket_name = DEFAULT_BUCKET_NAME <EOL> if not use_default_bucket and ud . has_key ( '<STR_LIT>' ) and ud [ '<STR_LIT>' ] is not None and ud [ '<STR_LIT>' ] is not None : <EOL> s3_conn = _get_s3_conn ( ud ) <EOL> if not _bucket_exists ( s3_conn , ud [ '<STR_LIT>' ] ) or not _remote_file_exists ( s3_conn , ud [ '<STR_LIT>' ] , ud [ '<STR_LIT>' ] ) : <EOL> log . debug ( "<STR_LIT>" % default_bucket_name ) <EOL> use_default_bucket = True <EOL> else : <EOL> log . debug ( "<STR_LIT>" % ud [ '<STR_LIT>' ] ) <EOL> use_default_bucket = False <EOL> else : <EOL> log . debug ( "<STR_LIT>" <EOL> % default_bucket_name ) <EOL> use_default_bucket = True <EOL> got_boot_script = False <EOL> if use_default_bucket is False : <EOL> log . debug ( "<STR_LIT>" <EOL> % ( ud [ '<STR_LIT>' ] , ud . get ( '<STR_LIT>' , None ) ) ) <EOL> got_boot_script = _get_file_from_bucket ( s3_conn , ud [ '<STR_LIT>' ] , ud [ '<STR_LIT>' ] , <EOL> DEFAULT_BOOT_SCRIPT_NAME ) <EOL> if got_boot_script : <EOL> os . chmod ( os . path . join ( LOCAL_PATH , DEFAULT_BOOT_SCRIPT_NAME ) , <NUM_LIT:0> <NUM_LIT> ) <EOL> if not got_boot_script or use_default_bucket : <EOL> boot_script_url = os . path . join ( _get_default_bucket_url ( ud ) , ud . get ( '<STR_LIT>' , <EOL> DEFAULT_BOOT_SCRIPT_NAME ) ) <EOL> log . debug ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( ud [ '<STR_LIT>' ] , ud . get ( '<STR_LIT>' , None ) , boot_script_url ) ) <EOL> got_boot_script = _get_file_from_url ( boot_script_url ) <EOL> if got_boot_script : <EOL> log . debug ( "<STR_LIT>" % os . path . join ( LOCAL_PATH , DEFAULT_BOOT_SCRIPT_NAME ) ) <EOL> use_object_store = ud . get ( "<STR_LIT>" , True ) <EOL> if use_object_store and ud . has_key ( '<STR_LIT>' ) and ud [ '<STR_LIT>' ] : <EOL> s3_conn = _get_s3_conn ( ud ) <EOL> if _bucket_exists ( s3_conn , ud [ '<STR_LIT>' ] ) and not _remote_file_exists ( s3_conn , ud [ '<STR_LIT>' ] , ud [ '<STR_LIT>' ] ) : <EOL> _save_file_to_bucket ( s3_conn , ud [ '<STR_LIT>' ] , ud [ '<STR_LIT>' ] , DEFAULT_BOOT_SCRIPT_NAME ) <EOL> return True <EOL> log . debug ( "<STR_LIT>" ) <EOL> return False <EOL> def _run_boot_script ( boot_script_name ) : <EOL> script = os . path . join ( LOCAL_PATH , boot_script_name ) <EOL> log . info ( "<STR_LIT>" % script ) <EOL> process = subprocess . Popen ( script , shell = True , stdout = subprocess . PIPE , stderr = subprocess . PIPE ) <EOL> stdout , stderr = process . communicate ( ) <EOL> if process . returncode == <NUM_LIT:0> : <EOL> log . debug ( "<STR_LIT>" % script ) <EOL> return True <EOL> else : <EOL> log . error ( "<STR_LIT>" <EOL> % ( script , process . returncode , stderr ) ) <EOL> return False <EOL> def _create_basic_user_data_file ( ) : <EOL> with open ( USER_DATA_FILE , '<STR_LIT:w>' ) as ud_file : <EOL> ud_formatted = { '<STR_LIT>' : None , <EOL> '<STR_LIT>' : DEFAULT_BOOT_SCRIPT_NAME , <EOL> '<STR_LIT>' : LOCAL_PATH , <EOL> '<STR_LIT>' : DEFAULT_BUCKET_NAME , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : CLOUDMAN_HOME , <EOL> '<STR_LIT>' : '<STR_LIT>' % random . randrange ( <NUM_LIT:1> , <NUM_LIT> ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : None } <EOL> yaml . dump ( ud_formatted , ud_file , default_flow_style = False ) <EOL> return ud_formatted <EOL> def _get_default_bucket_url ( ud = None ) : <EOL> if ud and ud . has_key ( '<STR_LIT>' ) : <EOL> default_bucket_name = ud [ '<STR_LIT>' ] <EOL> else : <EOL> default_bucket_name = DEFAULT_BUCKET_NAME <EOL> if ud : <EOL> bucket_url = ud . get ( "<STR_LIT>" , None ) <EOL> else : <EOL> bucket_url = None <EOL> if not bucket_url : <EOL> bucket_url = os . path . join ( SERVICE_ROOT , default_bucket_name ) <EOL> log . debug ( "<STR_LIT>" % bucket_url ) <EOL> return bucket_url <EOL> def _user_exists ( username ) : <EOL> """<STR_LIT>""" <EOL> with open ( '<STR_LIT>' , '<STR_LIT:r>' ) as f : <EOL> ep = f . read ( ) <EOL> return ep . find ( username ) > <NUM_LIT:0> <EOL> def _allow_password_logins ( passwd ) : <EOL> for user in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> if _user_exists ( user ) : <EOL> log . info ( "<STR_LIT>" . format ( user ) ) <EOL> p1 = subprocess . Popen ( [ "<STR_LIT>" , "<STR_LIT>" % ( user , passwd ) ] , stdout = subprocess . PIPE ) <EOL> p2 = subprocess . Popen ( [ "<STR_LIT>" ] , stdin = p1 . stdout , stdout = subprocess . PIPE ) <EOL> p1 . stdout . close ( ) <EOL> p2 . communicate ( ) [ <NUM_LIT:0> ] <EOL> cl = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] <EOL> subprocess . check_call ( cl ) <EOL> cl = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> subprocess . check_call ( cl ) <EOL> def _handle_freenx ( passwd ) : <EOL> cl = "<STR_LIT>" <EOL> retcode = subprocess . call ( cl , shell = True ) <EOL> if retcode == <NUM_LIT:0> : <EOL> log . info ( "<STR_LIT>" ) <EOL> cl = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> for i in range ( <NUM_LIT:5> ) : <EOL> retcode = subprocess . call ( cl ) <EOL> if retcode == <NUM_LIT:0> : <EOL> break <EOL> else : <EOL> time . sleep ( <NUM_LIT:5> ) <EOL> else : <EOL> log . info ( "<STR_LIT>" ) <EOL> def _handle_empty ( ) : <EOL> log . info ( "<STR_LIT>" ) <EOL> _create_basic_user_data_file ( ) <EOL> file_url = os . path . join ( _get_default_bucket_url ( ) , DEFAULT_BOOT_SCRIPT_NAME ) <EOL> log . debug ( "<STR_LIT>" % file_url ) <EOL> _get_file_from_url ( file_url ) <EOL> _run_boot_script ( DEFAULT_BOOT_SCRIPT_NAME ) <EOL> def _handle_url ( url ) : <EOL> log . info ( "<STR_LIT>" % url ) <EOL> _get_file_from_url ( url ) <EOL> boot_script_name = os . path . split ( url ) [ <NUM_LIT:1> ] <EOL> _run_boot_script ( boot_script_name ) <EOL> def _merge ( specific , default ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( specific , dict ) and isinstance ( default , dict ) : <EOL> for k , v in default . iteritems ( ) : <EOL> if k not in specific : <EOL> specific [ k ] = v <EOL> else : <EOL> specific [ k ] = _merge ( specific [ k ] , v ) <EOL> return specific <EOL> def _load_user_data ( user_data ) : <EOL> """<STR_LIT>""" <EOL> ud = yaml . load ( user_data ) <EOL> if ud == user_data : <EOL> return ud <EOL> default_user_data_path = os . path . join ( os . path . dirname ( os . path . abspath ( __file__ ) ) , '<STR_LIT>' ) <EOL> if os . path . exists ( default_user_data_path ) : <EOL> image_ud = yaml . load ( open ( default_user_data_path , '<STR_LIT:r>' ) . read ( ) ) <EOL> if image_ud : <EOL> ud = _merge ( ud , image_ud ) <EOL> return ud <EOL> def _handle_yaml ( user_data ) : <EOL> """<STR_LIT>""" <EOL> log . info ( "<STR_LIT>" ) <EOL> ud = _load_user_data ( user_data ) <EOL> if ud == user_data : <EOL> return _handle_empty ( ) <EOL> if "<STR_LIT>" in ud or "<STR_LIT:password>" in ud : <EOL> passwd = ud . get ( "<STR_LIT>" , None ) or ud . get ( "<STR_LIT:password>" , None ) <EOL> _allow_password_logins ( passwd ) <EOL> if "<STR_LIT>" in ud : <EOL> _handle_freenx ( ud [ "<STR_LIT>" ] ) <EOL> if len ( ud ) == <NUM_LIT:1> : <EOL> return _handle_empty ( ) <EOL> if ud . get ( '<STR_LIT>' , None ) is not None : <EOL> log . info ( "<STR_LIT>" ) <EOL> return <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> log . warning ( "<STR_LIT>" ) <EOL> ud [ '<STR_LIT>' ] = '<STR_LIT>' % random . randrange ( <NUM_LIT:1> , <NUM_LIT> ) <EOL> elif ud [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> log . warning ( "<STR_LIT>" ) <EOL> ud [ '<STR_LIT>' ] = '<STR_LIT>' % random . randrange ( <NUM_LIT:1> , <NUM_LIT> ) <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> log . info ( "<STR_LIT>" ) <EOL> ud [ '<STR_LIT>' ] = None <EOL> elif ud [ '<STR_LIT>' ] == '<STR_LIT>' or ud [ '<STR_LIT>' ] is None : <EOL> log . warning ( "<STR_LIT>" ) <EOL> ud [ '<STR_LIT>' ] = None <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> log . info ( "<STR_LIT>" ) <EOL> ud [ '<STR_LIT>' ] = None <EOL> elif ud [ '<STR_LIT>' ] == '<STR_LIT>' or ud [ '<STR_LIT>' ] is None : <EOL> log . warning ( "<STR_LIT>" ) <EOL> ud [ '<STR_LIT>' ] = None <EOL> if not ud . has_key ( '<STR_LIT:password>' ) : <EOL> log . warning ( "<STR_LIT>" ) <EOL> elif ud [ '<STR_LIT:password>' ] == '<STR_LIT>' : <EOL> log . warning ( "<STR_LIT>" ) <EOL> else : <EOL> ud [ '<STR_LIT:password>' ] = str ( ud [ '<STR_LIT:password>' ] ) <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> log . debug ( "<STR_LIT>" <EOL> % DEFAULT_BUCKET_NAME ) <EOL> ud [ '<STR_LIT>' ] = DEFAULT_BUCKET_NAME <EOL> elif ud [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> log . warning ( "<STR_LIT>" <EOL> % DEFAULT_BUCKET_NAME ) <EOL> ud [ '<STR_LIT>' ] = DEFAULT_BUCKET_NAME <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> if ud [ '<STR_LIT>' ] is not None and ud [ '<STR_LIT>' ] is not None : <EOL> ud [ '<STR_LIT>' ] = _get_bucket_name ( ud [ '<STR_LIT>' ] , ud [ '<STR_LIT>' ] ) <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> ud [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> ud [ '<STR_LIT>' ] = CLOUDMAN_HOME <EOL> if not ud . has_key ( '<STR_LIT>' ) : <EOL> ud [ '<STR_LIT>' ] = DEFAULT_BOOT_SCRIPT_NAME <EOL> ud [ '<STR_LIT>' ] = LOCAL_PATH <EOL> log . debug ( "<STR_LIT>" % ud ) <EOL> with open ( USER_DATA_FILE , '<STR_LIT:w>' ) as ud_yaml : <EOL> yaml . dump ( ud , ud_yaml , default_flow_style = False ) <EOL> if _get_boot_script ( ud ) : <EOL> _run_boot_script ( DEFAULT_BOOT_SCRIPT_NAME ) <EOL> def _parse_user_data ( ud ) : <EOL> if ud == '<STR_LIT>' : <EOL> _handle_empty ( ) <EOL> elif _isurl ( ud ) : <EOL> _handle_url ( ud ) <EOL> else : <EOL> _handle_yaml ( ud ) <EOL> def main ( ) : <EOL> if not os . path . exists ( LOCAL_PATH ) : <EOL> os . mkdir ( LOCAL_PATH ) <EOL> global log <EOL> log = _setup_logging ( ) <EOL> ud = _get_user_data ( ) <EOL> _parse_user_data ( ud ) <EOL> log . info ( "<STR_LIT>" % sys . argv [ <NUM_LIT:0> ] ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
