<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import os <EOL> import sys <EOL> pp_not_found_error = """<STR_LIT>""" <EOL> class TaskRunner : <EOL> def __init__ ( self , <EOL> task , <EOL> parallel = False , <EOL> num_workers = <NUM_LIT:0> , <EOL> chunk = <NUM_LIT> , <EOL> servers = None , <EOL> module_deps = None , <EOL> function_deps = None , <EOL> verbose = False ) : <EOL> """<STR_LIT>""" <EOL> self . task = task <EOL> self . parallel = parallel <EOL> self . num_workers = num_workers or "<STR_LIT>" <EOL> self . servers = servers or ( ) <EOL> self . chunk = chunk <EOL> if function_deps : <EOL> self . function_deps = tuple ( function_deps ) <EOL> else : <EOL> self . function_deps = ( ) <EOL> if module_deps : <EOL> self . module_deps = tuple ( module_deps ) <EOL> else : <EOL> self . module_deps = ( ) <EOL> self . verbose = verbose <EOL> if parallel : <EOL> try : <EOL> import pp <EOL> except ImportError as e : <EOL> sys . stderr . write ( pp_not_found_error ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> self . pp = pp <EOL> return <EOL> def __call__ ( self , * args ) : <EOL> if self . parallel : <EOL> args = list ( args ) <EOL> if type ( args [ <NUM_LIT:0> ] ) is not list : <EOL> args [ <NUM_LIT:0> ] = list ( args [ <NUM_LIT:0> ] ) <EOL> return self . RunParallel ( args ) <EOL> else : <EOL> return self . RunNormal ( args ) <EOL> def RunNormal ( self , args ) : <EOL> """<STR_LIT>""" <EOL> return self . task ( * args ) <EOL> def RunParallel ( self , args ) : <EOL> job_server = self . pp . Server ( <EOL> ncpus = self . num_workers , <EOL> ppservers = self . servers , <EOL> secret = '<STR_LIT>' ) <EOL> if self . verbose : <EOL> sys . stderr . write ( "<STR_LIT>" % ( job_server . get_ncpus ( ) ) ) <EOL> jobs = [ ] <EOL> load = args [ <NUM_LIT:0> ] <EOL> dispatched = <NUM_LIT:0> <EOL> while dispatched < len ( load ) : <EOL> args [ <NUM_LIT:0> ] = load [ dispatched : dispatched + self . chunk ] <EOL> dispatched += self . chunk <EOL> jobs . append ( job_server . submit ( <EOL> self . task , <EOL> tuple ( args ) , <EOL> self . function_deps , <EOL> self . module_deps , <EOL> ) ) <EOL> results = [ job ( ) for job in jobs ] <EOL> if self . verbose : <EOL> fp = sys . stdout <EOL> sys . stdout = sys . stderr <EOL> job_server . print_stats ( ) <EOL> sys . stdout = fp <EOL> job_server . destroy ( ) <EOL> return results </s>
