<s> import git <EOL> import logging <EOL> import magic <EOL> import os <EOL> import requests <EOL> import select <EOL> import shutil <EOL> import subprocess <EOL> import swiftclient <EOL> import sys <EOL> import tempfile <EOL> import time <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> class GitRepository ( object ) : <EOL> """<STR_LIT>""" <EOL> log = logging . getLogger ( "<STR_LIT>" ) <EOL> def __init__ ( self , remote_url , local_path ) : <EOL> self . remote_url = remote_url <EOL> self . local_path = local_path <EOL> self . _ensure_cloned ( ) <EOL> self . repo = git . Repo ( self . local_path ) <EOL> def _ensure_cloned ( self ) : <EOL> if not os . path . exists ( self . local_path ) : <EOL> self . log . debug ( "<STR_LIT>" % ( self . remote_url , <EOL> self . local_path ) ) <EOL> git . Repo . clone_from ( self . remote_url , self . local_path ) <EOL> def fetch ( self , ref ) : <EOL> origin = self . repo . remotes . origin <EOL> self . log . debug ( "<STR_LIT>" % ( ref , origin ) ) <EOL> try : <EOL> origin . fetch ( ref ) <EOL> except AssertionError : <EOL> origin . fetch ( ref ) <EOL> def checkout ( self , ref ) : <EOL> self . log . debug ( "<STR_LIT>" % ref ) <EOL> return self . repo . git . checkout ( ref ) <EOL> def reset ( self ) : <EOL> self . _ensure_cloned ( ) <EOL> self . log . debug ( "<STR_LIT>" % self . local_path ) <EOL> self . update ( ) <EOL> origin = self . repo . remotes . origin <EOL> for ref in origin . refs : <EOL> if ref . remote_head == '<STR_LIT>' : <EOL> continue <EOL> self . repo . create_head ( ref . remote_head , ref , force = True ) <EOL> self . repo . head . reference = origin . refs [ '<STR_LIT>' ] <EOL> self . repo . head . reset ( index = True , working_tree = True ) <EOL> self . repo . git . clean ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def update ( self ) : <EOL> self . _ensure_cloned ( ) <EOL> self . log . debug ( "<STR_LIT>" % self . local_path ) <EOL> origin = self . repo . remotes . origin <EOL> origin . update ( ) <EOL> self . repo = git . Repo ( self . local_path ) <EOL> def execute_to_log ( cmd , logfile , timeout = - <NUM_LIT:1> , watch_logs = [ ] , heartbeat = <NUM_LIT:30> , <EOL> env = None , cwd = None ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( os . path . dirname ( logfile ) ) : <EOL> os . makedirs ( os . path . dirname ( logfile ) ) <EOL> logger = logging . getLogger ( logfile ) <EOL> log_handler = logging . FileHandler ( logfile ) <EOL> log_formatter = logging . Formatter ( '<STR_LIT>' ) <EOL> log_handler . setFormatter ( log_formatter ) <EOL> logger . addHandler ( log_handler ) <EOL> descriptors = { } <EOL> for watch_file in watch_logs : <EOL> if not os . path . exists ( watch_file [ <NUM_LIT:1> ] ) : <EOL> logger . warning ( '<STR_LIT>' <EOL> % watch_file [ <NUM_LIT:1> ] ) <EOL> continue <EOL> try : <EOL> fd = os . open ( watch_file [ <NUM_LIT:1> ] , os . O_RDONLY ) <EOL> os . lseek ( fd , <NUM_LIT:0> , os . SEEK_END ) <EOL> descriptors [ fd ] = { '<STR_LIT:name>' : watch_file [ <NUM_LIT:0> ] , <EOL> '<STR_LIT>' : select . POLLIN , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> except Exception as e : <EOL> logger . warning ( '<STR_LIT>' <EOL> % ( watch_file [ <NUM_LIT:1> ] , e ) ) <EOL> cmd += '<STR_LIT>' <EOL> logger . info ( "<STR_LIT>" % cmd ) <EOL> start_time = time . time ( ) <EOL> p = subprocess . Popen ( <EOL> cmd , shell = True , stdout = subprocess . PIPE , stderr = subprocess . PIPE , <EOL> env = env , cwd = cwd ) <EOL> descriptors [ p . stdout . fileno ( ) ] = dict ( <EOL> name = '<STR_LIT>' , <EOL> poll = ( select . POLLIN | select . POLLHUP ) , <EOL> lines = '<STR_LIT>' <EOL> ) <EOL> poll_obj = select . poll ( ) <EOL> for fd , descriptor in descriptors . items ( ) : <EOL> poll_obj . register ( fd , descriptor [ '<STR_LIT>' ] ) <EOL> last_heartbeat = time . time ( ) <EOL> def process ( fd ) : <EOL> """<STR_LIT>""" <EOL> global last_heartbeat <EOL> descriptors [ fd ] [ '<STR_LIT>' ] += os . read ( fd , <NUM_LIT> * <NUM_LIT> ) <EOL> if descriptors [ fd ] [ '<STR_LIT>' ] . find ( '<STR_LIT:\n>' ) != - <NUM_LIT:1> : <EOL> elems = descriptors [ fd ] [ '<STR_LIT>' ] . split ( '<STR_LIT:\n>' ) <EOL> for l in elems [ : - <NUM_LIT:1> ] : <EOL> if len ( l ) > <NUM_LIT:0> : <EOL> l = '<STR_LIT>' % ( descriptors [ fd ] [ '<STR_LIT:name>' ] , l ) <EOL> logger . info ( l ) <EOL> last_heartbeat = time . time ( ) <EOL> descriptors [ fd ] [ '<STR_LIT>' ] = elems [ - <NUM_LIT:1> ] <EOL> while p . poll ( ) is None : <EOL> if timeout > <NUM_LIT:0> and time . time ( ) - start_time > timeout : <EOL> logger . info ( "<STR_LIT>" ) <EOL> os . kill ( p . pid , <NUM_LIT:9> ) <EOL> for fd , flag in poll_obj . poll ( <NUM_LIT:0> ) : <EOL> process ( fd ) <EOL> if heartbeat and ( time . time ( ) - last_heartbeat > heartbeat ) : <EOL> logger . info ( "<STR_LIT>" ) <EOL> last_heartbeat = time . time ( ) <EOL> for fd , flag in poll_obj . poll ( <NUM_LIT:0> ) : <EOL> process ( fd ) <EOL> for fd , descriptor in descriptors . items ( ) : <EOL> poll_obj . unregister ( fd ) <EOL> if fd == p . stdout . fileno ( ) : <EOL> continue <EOL> os . close ( fd ) <EOL> try : <EOL> p . kill ( ) <EOL> except OSError : <EOL> pass <EOL> logger . info ( '<STR_LIT>' % p . returncode ) <EOL> logger . removeHandler ( log_handler ) <EOL> log_handler . flush ( ) <EOL> log_handler . close ( ) <EOL> return p . returncode <EOL> def zuul_swift_upload ( file_path , job_arguments ) : <EOL> """<STR_LIT>""" <EOL> file_list = [ ] <EOL> if os . path . isfile ( file_path ) : <EOL> file_list . append ( file_path ) <EOL> elif os . path . isdir ( file_path ) : <EOL> for path , folders , files in os . walk ( file_path ) : <EOL> for f in files : <EOL> f_path = os . path . join ( path , f ) <EOL> file_list . append ( f_path ) <EOL> payload = { } <EOL> ( object_prefix , <EOL> payload [ '<STR_LIT>' ] , <EOL> payload [ '<STR_LIT>' ] , <EOL> payload [ '<STR_LIT>' ] , <EOL> payload [ '<STR_LIT>' ] ) = job_arguments [ '<STR_LIT>' ] . split ( '<STR_LIT:\n>' ) <EOL> url = job_arguments [ '<STR_LIT>' ] <EOL> payload [ '<STR_LIT>' ] = job_arguments [ '<STR_LIT>' ] <EOL> logserver_prefix = job_arguments [ '<STR_LIT>' ] <EOL> files = { } <EOL> for i , f in enumerate ( file_list ) : <EOL> files [ '<STR_LIT>' % ( i + <NUM_LIT:1> ) ] = open ( f , '<STR_LIT:rb>' ) <EOL> requests . post ( url , data = payload , files = files ) <EOL> return ( logserver_prefix + <EOL> job_arguments [ '<STR_LIT>' ] ) <EOL> def generate_log_index ( file_list , logserver_prefix , results_set_name ) : <EOL> """<STR_LIT>""" <EOL> output = '<STR_LIT>' <EOL> output += '<STR_LIT>' <EOL> for f in file_list : <EOL> file_url = os . path . join ( logserver_prefix , results_set_name , f ) <EOL> filename_postfix = '<STR_LIT:/>' if f [ - <NUM_LIT:1> ] == '<STR_LIT:/>' else '<STR_LIT>' <EOL> filename = os . path . basename ( os . path . normpath ( f ) ) + filename_postfix <EOL> output += '<STR_LIT>' <EOL> output += '<STR_LIT>' % ( file_url , filename ) <EOL> output += '<STR_LIT>' <EOL> output += '<STR_LIT>' <EOL> output += '<STR_LIT>' <EOL> return output <EOL> def make_index_file ( file_list , logserver_prefix , results_set_name , <EOL> index_filename = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> index_content = generate_log_index ( file_list , logserver_prefix , <EOL> results_set_name ) <EOL> tempdir = tempfile . mkdtemp ( ) <EOL> fd = open ( os . path . join ( tempdir , index_filename ) , '<STR_LIT:w>' ) <EOL> fd . write ( index_content ) <EOL> return os . path . join ( tempdir , index_filename ) <EOL> def get_file_mime ( file_path ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isfile ( file_path ) : <EOL> return None <EOL> if hasattr ( magic , '<STR_LIT>' ) : <EOL> return magic . from_file ( file_path , mime = True ) <EOL> else : <EOL> m = magic . open ( magic . MAGIC_MIME ) <EOL> m . load ( ) <EOL> return m . file ( file_path ) . split ( '<STR_LIT:;>' ) [ <NUM_LIT:0> ] <EOL> def swift_form_post_submit ( file_list , url , hmac_body , signature ) : <EOL> """<STR_LIT>""" <EOL> payload = { } <EOL> ( object_prefix , <EOL> payload [ '<STR_LIT>' ] , <EOL> payload [ '<STR_LIT>' ] , <EOL> payload [ '<STR_LIT>' ] , <EOL> payload [ '<STR_LIT>' ] ) = hmac_body . split ( '<STR_LIT:\n>' ) <EOL> payload [ '<STR_LIT>' ] = signature <EOL> for sub_file_list in ( file_list [ pos : pos + int ( payload [ '<STR_LIT>' ] ) ] <EOL> for pos in xrange ( <NUM_LIT:0> , len ( file_list ) , <EOL> int ( payload [ '<STR_LIT>' ] ) ) ) : <EOL> if payload [ '<STR_LIT>' ] < time . time ( ) : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> files = { } <EOL> filename_prefix = '<STR_LIT:/>' if url [ - <NUM_LIT:1> ] != '<STR_LIT:/>' else '<STR_LIT>' <EOL> for i , f in enumerate ( sub_file_list ) : <EOL> if os . path . getsize ( f [ '<STR_LIT:path>' ] ) > int ( payload [ '<STR_LIT>' ] ) : <EOL> sys . stderr . write ( '<STR_LIT>' <EOL> % ( f [ '<STR_LIT:path>' ] , int ( payload [ '<STR_LIT>' ] ) ) ) <EOL> continue <EOL> files [ '<STR_LIT>' % ( i + <NUM_LIT:1> ) ] = ( filename_prefix + f [ '<STR_LIT:filename>' ] , <EOL> open ( f [ '<STR_LIT:path>' ] , '<STR_LIT:rb>' ) , <EOL> get_file_mime ( f [ '<STR_LIT:path>' ] ) ) <EOL> requests . post ( url , data = payload , files = files ) <EOL> def build_file_list ( file_path , logserver_prefix , results_set_name , <EOL> create_dir_indexes = True ) : <EOL> """<STR_LIT>""" <EOL> file_list = [ ] <EOL> if os . path . isfile ( file_path ) : <EOL> file_list . append ( { '<STR_LIT:filename>' : os . path . basename ( file_path ) , <EOL> '<STR_LIT:path>' : file_path } ) <EOL> elif os . path . isdir ( file_path ) : <EOL> if file_path [ - <NUM_LIT:1> ] == os . sep : <EOL> file_path = file_path [ : - <NUM_LIT:1> ] <EOL> parent_dir = os . path . dirname ( file_path ) <EOL> for path , folders , files in os . walk ( file_path ) : <EOL> folder_contents = [ ] <EOL> for f in files : <EOL> full_path = os . path . join ( path , f ) <EOL> relative_name = os . path . relpath ( full_path , parent_dir ) <EOL> push_file = { '<STR_LIT:filename>' : relative_name , <EOL> '<STR_LIT:path>' : full_path } <EOL> file_list . append ( push_file ) <EOL> folder_contents . append ( relative_name ) <EOL> for f in folders : <EOL> full_path = os . path . join ( path , f ) <EOL> relative_name = os . path . relpath ( full_path , parent_dir ) <EOL> folder_contents . append ( relative_name + '<STR_LIT:/>' ) <EOL> if create_dir_indexes : <EOL> index_file = make_index_file ( folder_contents , logserver_prefix , <EOL> results_set_name ) <EOL> relative_name = os . path . relpath ( path , parent_dir ) <EOL> file_list . append ( { <EOL> '<STR_LIT:filename>' : os . path . join ( relative_name , <EOL> os . path . basename ( index_file ) ) , <EOL> '<STR_LIT:path>' : index_file } ) <EOL> return file_list <EOL> def push_files ( results_set_name , path_list , publish_config , <EOL> generate_indexes = True ) : <EOL> """<STR_LIT>""" <EOL> file_list = [ ] <EOL> root_list = [ ] <EOL> for file_path in path_list : <EOL> file_path = os . path . normpath ( file_path ) <EOL> if os . path . isfile ( file_path ) : <EOL> root_list . append ( os . path . basename ( file_path ) ) <EOL> else : <EOL> root_list . append ( os . path . basename ( file_path ) + '<STR_LIT:/>' ) <EOL> file_list += build_file_list ( <EOL> file_path , publish_config [ '<STR_LIT>' ] , results_set_name , <EOL> generate_indexes <EOL> ) <EOL> index_file = '<STR_LIT>' <EOL> if generate_indexes : <EOL> index_file = make_index_file ( root_list , publish_config [ '<STR_LIT>' ] , <EOL> results_set_name ) <EOL> file_list . append ( { <EOL> '<STR_LIT:filename>' : os . path . basename ( index_file ) , <EOL> '<STR_LIT:path>' : index_file } ) <EOL> method = publish_config [ '<STR_LIT:type>' ] + '<STR_LIT>' <EOL> if method in globals ( ) and hasattr ( globals ( ) [ method ] , '<STR_LIT>' ) : <EOL> globals ( ) [ method ] ( results_set_name , file_list , publish_config ) <EOL> return os . path . join ( publish_config [ '<STR_LIT>' ] , results_set_name , <EOL> os . path . basename ( index_file ) ) <EOL> def swift_push_files ( results_set_name , file_list , swift_config ) : <EOL> """<STR_LIT>""" <EOL> for file_item in file_list : <EOL> with open ( file_item [ '<STR_LIT:path>' ] , '<STR_LIT:r>' ) as fd : <EOL> con = swiftclient . client . Connection ( <EOL> authurl = swift_config [ '<STR_LIT>' ] , <EOL> user = swift_config [ '<STR_LIT:user>' ] , <EOL> key = swift_config [ '<STR_LIT:password>' ] , <EOL> os_options = { '<STR_LIT>' : swift_config [ '<STR_LIT>' ] } , <EOL> tenant_name = swift_config [ '<STR_LIT>' ] , <EOL> auth_version = <NUM_LIT> ) <EOL> filename = os . path . join ( results_set_name , file_item [ '<STR_LIT:filename>' ] ) <EOL> con . put_object ( swift_config [ '<STR_LIT>' ] , filename , fd ) <EOL> def local_push_files ( results_set_name , file_list , local_config ) : <EOL> """<STR_LIT>""" <EOL> for file_item in file_list : <EOL> dest_dir = os . path . join ( local_config [ '<STR_LIT:path>' ] , results_set_name , <EOL> os . path . dirname ( file_item [ '<STR_LIT:filename>' ] ) ) <EOL> dest_filename = os . path . basename ( file_item [ '<STR_LIT:filename>' ] ) <EOL> if not os . path . isdir ( dest_dir ) : <EOL> os . makedirs ( dest_dir ) <EOL> dest_file = os . path . join ( dest_dir , dest_filename ) <EOL> shutil . copyfile ( file_item [ '<STR_LIT:path>' ] , dest_file ) <EOL> def scp_push_files ( results_set_name , file_path , local_config ) : <EOL> """<STR_LIT>""" <EOL> pass </s>
