<s> '''<STR_LIT>''' <EOL> import sys <EOL> import os <EOL> import re <EOL> import subprocess <EOL> import optparse <EOL> import tempfile <EOL> import collections <EOL> import itertools <EOL> import random <EOL> from multiprocessing import Process <EOL> from threading import Thread as Process <EOL> import CGAT . Experiment as E <EOL> import CGAT . Stats as Stats <EOL> import CGAT . Logfile as Logfile <EOL> CODE_CPU = """<STR_LIT>""" <EOL> CODE_SCRIPT = """<STR_LIT>""" <EOL> HOST = collections . namedtuple ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> class Host : pass <EOL> def getSGEHosts ( ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> x = subprocess . Popen ( "<STR_LIT>" , <EOL> shell = True , <EOL> stdout = subprocess . PIPE , <EOL> ) <EOL> except OSError , e : <EOL> E . fail ( "<STR_LIT>" % cmd ) <EOL> raise <EOL> hosts = [ ] <EOL> for line in x . stdout : <EOL> if line . startswith ( "<STR_LIT>" ) : continue <EOL> if line . startswith ( "<STR_LIT>" ) : continue <EOL> if line . startswith ( "<STR_LIT:->" ) : continue <EOL> x = HOST ( * re . split ( "<STR_LIT>" , line [ : - <NUM_LIT:1> ] ) ) <EOL> if x . ncpu == "<STR_LIT:->" or x . arch == "<STR_LIT:->" : continue <EOL> try : x = x . _replace ( ncpu = int ( x . ncpu ) , ) <EOL> except ValueError : pass <EOL> assert x . memtot . endswith ( "<STR_LIT>" ) <EOL> x = x . _replace ( memtot = float ( x . memtot [ : - <NUM_LIT:1> ] ) ) <EOL> hosts . append ( x ) <EOL> return hosts <EOL> def writeResults ( outfile , results ) : <EOL> fields = ( "<STR_LIT>" , "<STR_LIT:user>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ) <EOL> outfile . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( [ "<STR_LIT>" % ( x , y ) for x , y in itertools . product ( fields , Stats . Summary ( ) . getHeaders ( ) ) ] ) ) <EOL> hosts = results . keys ( ) <EOL> hosts . sort ( ) <EOL> for host in hosts : <EOL> result = results [ host ] <EOL> outfile . write ( "<STR_LIT:%s>" % host ) <EOL> for f in fields : <EOL> d = [ y . __getitem__ ( f ) for y in result ] <EOL> outfile . write ( "<STR_LIT>" % Stats . Summary ( d ) ) <EOL> outfile . write ( "<STR_LIT:\n>" ) <EOL> def main ( ) : <EOL> parser = optparse . OptionParser ( version = "<STR_LIT>" , usage = globals ( ) [ "<STR_LIT>" ] ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , type = "<STR_LIT>" , <EOL> choices = ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> help = "<STR_LIT>" ) <EOL> parser . set_defaults ( <EOL> iterations_cpu = <NUM_LIT> , <EOL> iterations_sample = <NUM_LIT:5> , <EOL> saturation = <NUM_LIT:2> , <EOL> host_block = <NUM_LIT:10> , <EOL> suite = "<STR_LIT>" , <EOL> cluster_prefix = """<STR_LIT>""" , <EOL> ) <EOL> ( options , args ) = E . Start ( parser , add_cluster_options = True ) <EOL> hosts = [ x for x in getSGEHosts ( ) if x . ncpu != "<STR_LIT:->" ] <EOL> E . info ( "<STR_LIT>" % len ( hosts ) ) <EOL> curdir = os . path . abspath ( os . getcwd ( ) ) <EOL> outfile = options . stdout <EOL> if options . suite == "<STR_LIT>" : <EOL> outfile . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( HOST . _fields ) ) <EOL> for host in hosts : <EOL> outfile . write ( "<STR_LIT>" % "<STR_LIT:\t>" . join ( map ( str , host ) ) ) <EOL> elif suite == "<STR_LIT>" : <EOL> pythonscript = os . path . join ( curdir , "<STR_LIT>" ) <EOL> if os . path . exists ( pythonscript ) : <EOL> raise OSError ( "<STR_LIT>" % pythonscript ) <EOL> with open ( pythonscript , "<STR_LIT:w>" ) as outfile : <EOL> if options . suite == "<STR_LIT>" : <EOL> iterations = options . iterations_cpu <EOL> outfile . write ( CODE_CPU % locals ( ) ) <EOL> results = collections . defaultdict ( list ) <EOL> def run ( script , host , options ) : <EOL> """<STR_LIT>""" <EOL> queue = options . cluster_queue <EOL> command = options . cluster_prefix % locals ( ) <EOL> E . debug ( "<STR_LIT>" % command ) <EOL> try : <EOL> child = subprocess . Popen ( command , <EOL> shell = True , <EOL> stdout = subprocess . PIPE , <EOL> stdin = None , <EOL> close_fds = True ) <EOL> ( stdout , stderr ) = child . communicate ( ) <EOL> except OSError , e : <EOL> E . warn ( "<STR_LIT>" % cmd ) <EOL> raise <EOL> E . debug ( "<STR_LIT>" % command ) <EOL> data = Logfile . LogFileData ( ) <EOL> for line in stdout . split ( "<STR_LIT:\n>" ) : <EOL> data . add ( line ) <EOL> results [ host ] . append ( data ) <EOL> idx = <NUM_LIT:0> <EOL> for n in range ( <NUM_LIT:0> , options . iterations_sample ) : <EOL> myhosts = list ( hosts [ : ] ) <EOL> random . shuffle ( myhosts ) <EOL> for h in range ( <NUM_LIT:0> , len ( myhosts ) , options . host_block ) : <EOL> for host in ( myhosts [ x ] for x in range ( h , min ( len ( myhosts ) , h + options . host_block ) ) ) : <EOL> processes = [ ] <EOL> for cpu in range ( <NUM_LIT:0> , int ( host . ncpu * options . saturation ) ) : <EOL> p = Process ( target = run , args = ( pythonscript , host . host , options ) ) <EOL> processes . append ( p ) <EOL> p . start ( ) <EOL> E . info ( "<STR_LIT>" % ( n , host . host , len ( processes ) ) ) <EOL> for p in processes : p . join ( ) <EOL> filename = "<STR_LIT>" % idx <EOL> E . info ( "<STR_LIT>" % ( n , filename ) ) <EOL> idx += <NUM_LIT:1> <EOL> outfile = open ( filename , "<STR_LIT:w>" ) <EOL> writeResults ( outfile , results ) <EOL> outfile . close ( ) <EOL> writeResults ( options . stdout , results ) <EOL> E . Stop ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( main ( ) ) </s>
