<s> """<STR_LIT>""" <EOL> import glob <EOL> import logging <EOL> import os . path <EOL> import shutil <EOL> import stat <EOL> import sys <EOL> import time <EOL> from openmdao . main . datatypes . api import Bool , Dict , Str , FileRef , Float , Int , List <EOL> from openmdao . main . api import Component <EOL> from openmdao . main . exceptions import RunInterrupted , RunStopped <EOL> from openmdao . main . rbac import AccessController , RoleError , rbac , remote_access <EOL> from openmdao . main . resource import ResourceAllocationManager as RAM <EOL> from openmdao . util . filexfer import filexfer , pack_zipfile , unpack_zipfile <EOL> from openmdao . util import shellproc <EOL> from numpy . distutils . exec_command import find_executable <EOL> class ExternalCode ( Component ) : <EOL> """<STR_LIT>""" <EOL> STDOUT = shellproc . STDOUT <EOL> DEV_NULL = shellproc . DEV_NULL <EOL> command = List ( Str , desc = '<STR_LIT>' ) <EOL> env_vars = Dict ( { } , iotype = '<STR_LIT>' , <EOL> desc = '<STR_LIT>' ) <EOL> resources = Dict ( { } , iotype = '<STR_LIT>' , <EOL> desc = '<STR_LIT>' ) <EOL> poll_delay = Float ( <NUM_LIT:0.> , low = <NUM_LIT:0.> , units = '<STR_LIT:s>' , iotype = '<STR_LIT>' , <EOL> desc = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> timeout = Float ( <NUM_LIT:0.> , low = <NUM_LIT:0.> , iotype = '<STR_LIT>' , units = '<STR_LIT:s>' , <EOL> desc = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> timed_out = Bool ( False , iotype = '<STR_LIT>' , desc = '<STR_LIT>' ) <EOL> return_code = Int ( <NUM_LIT:0> , iotype = '<STR_LIT>' , desc = '<STR_LIT>' ) <EOL> def __init__ ( self ) : <EOL> super ( ExternalCode , self ) . __init__ ( ) <EOL> self . check_external_outputs = True <EOL> self . stdin = self . DEV_NULL <EOL> self . stdout = None <EOL> self . stderr = "<STR_LIT>" <EOL> self . _process = None <EOL> self . _server = None <EOL> def get_access_controller ( self ) : <EOL> """<STR_LIT>""" <EOL> return _AccessController ( ) <EOL> @ rbac ( ( '<STR_LIT>' , '<STR_LIT:user>' ) ) <EOL> def set ( self , path , value ) : <EOL> """<STR_LIT>""" <EOL> if path in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) and remote_access ( ) : <EOL> self . raise_exception ( '<STR_LIT>' % path , <EOL> RuntimeError ) <EOL> return super ( ExternalCode , self ) . set ( path , value ) <EOL> def execute ( self ) : <EOL> """<STR_LIT>""" <EOL> self . return_code = - <NUM_LIT> <EOL> self . timed_out = False <EOL> if not self . command : <EOL> self . raise_exception ( '<STR_LIT>' , ValueError ) <EOL> self . check_files ( inputs = True ) <EOL> return_code = None <EOL> error_msg = '<STR_LIT>' <EOL> try : <EOL> if self . resources : <EOL> return_code , error_msg = self . _execute_remote ( ) <EOL> else : <EOL> return_code , error_msg = self . _execute_local ( ) <EOL> if return_code is None : <EOL> if self . _stop : <EOL> self . raise_exception ( '<STR_LIT>' , RunStopped ) <EOL> else : <EOL> self . timed_out = True <EOL> self . raise_exception ( '<STR_LIT>' , RunInterrupted ) <EOL> elif return_code : <EOL> if isinstance ( self . stderr , str ) : <EOL> if os . path . exists ( self . stderr ) : <EOL> stderrfile = open ( self . stderr , '<STR_LIT:r>' ) <EOL> error_desc = stderrfile . read ( ) <EOL> stderrfile . close ( ) <EOL> err_fragment = "<STR_LIT>" % error_desc <EOL> else : <EOL> err_fragment = "<STR_LIT>" % self . stderr <EOL> else : <EOL> err_fragment = error_msg <EOL> self . raise_exception ( '<STR_LIT>' % ( return_code , err_fragment ) , RuntimeError ) <EOL> if self . check_external_outputs : <EOL> self . check_files ( inputs = False ) <EOL> finally : <EOL> self . return_code = - <NUM_LIT> if return_code is None else return_code <EOL> def check_files ( self , inputs ) : <EOL> """<STR_LIT>""" <EOL> for metadata in self . external_files : <EOL> path = metadata . path <EOL> for ch in ( '<STR_LIT>' ) : <EOL> if ch in path : <EOL> break <EOL> else : <EOL> if inputs : <EOL> if not metadata . get ( '<STR_LIT:input>' , False ) : <EOL> continue <EOL> else : <EOL> if not metadata . get ( '<STR_LIT>' , False ) : <EOL> continue <EOL> if not os . path . exists ( path ) : <EOL> iotype = '<STR_LIT:input>' if inputs else '<STR_LIT>' <EOL> self . raise_exception ( '<STR_LIT>' % ( iotype , path ) , <EOL> RuntimeError ) <EOL> if inputs and self . stdin and self . stdin != self . DEV_NULL : <EOL> if not os . path . exists ( self . stdin ) : <EOL> self . raise_exception ( '<STR_LIT>' % self . stdin , <EOL> RuntimeError ) <EOL> if not inputs and self . stdout and self . stdout != self . DEV_NULL : <EOL> if not os . path . exists ( self . stdout ) : <EOL> self . raise_exception ( '<STR_LIT>' % self . stdout , <EOL> RuntimeError ) <EOL> if not inputs and self . stderr and self . stderr != self . DEV_NULL and self . stderr != self . STDOUT and ( not self . resources or not self . resources . get ( '<STR_LIT>' ) ) : <EOL> if not os . path . exists ( self . stderr ) : <EOL> self . raise_exception ( '<STR_LIT>' % self . stderr , <EOL> RuntimeError ) <EOL> if inputs : <EOL> for pathname , obj in self . items ( iotype = '<STR_LIT>' , recurse = True ) : <EOL> if isinstance ( obj , FileRef ) : <EOL> path = self . get_metadata ( pathname , '<STR_LIT>' ) <EOL> if path and not os . path . exists ( path ) : <EOL> self . raise_exception ( "<STR_LIT>" % path , <EOL> RuntimeError ) <EOL> else : <EOL> for pathname , obj in self . items ( iotype = '<STR_LIT>' , recurse = True ) : <EOL> if isinstance ( obj , FileRef ) : <EOL> if not os . path . exists ( obj . path ) : <EOL> self . raise_exception ( "<STR_LIT>" % obj . path , <EOL> RuntimeError ) <EOL> def _execute_local ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _logger . info ( '<STR_LIT>' , self . command ) <EOL> start_time = time . time ( ) <EOL> if isinstance ( self . command , basestring ) : <EOL> program_to_execute = self . command <EOL> else : <EOL> program_to_execute = self . command [ <NUM_LIT:0> ] <EOL> command_full_path = find_executable ( program_to_execute ) <EOL> if not command_full_path : <EOL> self . raise_exception ( "<STR_LIT>" % program_to_execute , <EOL> ValueError ) <EOL> command_for_shell_proc = self . command <EOL> if sys . platform == '<STR_LIT:win32>' : <EOL> command_for_shell_proc = [ '<STR_LIT>' , '<STR_LIT>' ] + command_for_shell_proc <EOL> self . _process = shellproc . ShellProc ( command_for_shell_proc , self . stdin , <EOL> self . stdout , self . stderr , self . env_vars ) <EOL> self . _logger . debug ( '<STR_LIT>' , self . _process . pid ) <EOL> try : <EOL> return_code , error_msg = self . _process . wait ( self . poll_delay , self . timeout ) <EOL> finally : <EOL> self . _process . close_files ( ) <EOL> self . _process = None <EOL> et = time . time ( ) - start_time <EOL> if et >= <NUM_LIT> : <EOL> self . _logger . info ( '<STR_LIT>' , et ) <EOL> return ( return_code , error_msg ) <EOL> def _execute_remote ( self ) : <EOL> """<STR_LIT>""" <EOL> rdesc = self . resources . copy ( ) <EOL> self . _server , server_info = RAM . allocate ( rdesc ) <EOL> if self . _server is None : <EOL> self . raise_exception ( '<STR_LIT>' , RuntimeError ) <EOL> if self . _logger . level == logging . NOTSET : <EOL> self . _server . set_log_level ( logging . DEBUG ) <EOL> else : <EOL> self . _server . set_log_level ( self . _logger . level ) <EOL> return_code = - <NUM_LIT> <EOL> error_msg = '<STR_LIT>' <EOL> try : <EOL> rdesc [ '<STR_LIT>' ] = self . get_pathname ( ) or self . __class__ . __name__ <EOL> rdesc [ '<STR_LIT>' ] = self . command [ <NUM_LIT:0> ] <EOL> if len ( self . command ) > <NUM_LIT:1> : <EOL> rdesc [ '<STR_LIT:args>' ] = self . command [ <NUM_LIT:1> : ] <EOL> if self . env_vars : <EOL> rdesc [ '<STR_LIT>' ] = self . env_vars <EOL> if not self . stdin : <EOL> self . raise_exception ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % self . stdin , ValueError ) <EOL> if self . stdin != self . DEV_NULL : <EOL> rdesc [ '<STR_LIT>' ] = self . stdin <EOL> if self . stdout : <EOL> rdesc [ '<STR_LIT>' ] = self . stdout <EOL> else : <EOL> rdesc [ '<STR_LIT>' ] = '<STR_LIT>' % self . command [ <NUM_LIT:0> ] <EOL> if self . stderr : <EOL> if self . stderr == self . STDOUT : <EOL> rdesc [ '<STR_LIT>' ] = True <EOL> else : <EOL> rdesc [ '<STR_LIT>' ] = self . stderr <EOL> else : <EOL> rdesc [ '<STR_LIT>' ] = '<STR_LIT>' % self . command [ <NUM_LIT:0> ] <EOL> if self . timeout : <EOL> if '<STR_LIT>' in rdesc : <EOL> limits = rdesc [ '<STR_LIT>' ] . copy ( ) <EOL> else : <EOL> limits = { } <EOL> limits [ '<STR_LIT>' ] = self . timeout <EOL> rdesc [ '<STR_LIT>' ] = limits <EOL> patterns = [ ] <EOL> textfiles = [ ] <EOL> for metadata in self . external_files : <EOL> if metadata . get ( '<STR_LIT:input>' , False ) : <EOL> patterns . append ( metadata . path ) <EOL> if not metadata . binary : <EOL> textfiles . append ( metadata . path ) <EOL> for pathname , obj in self . items ( iotype = '<STR_LIT>' , recurse = True ) : <EOL> if isinstance ( obj , FileRef ) : <EOL> local_path = self . get_metadata ( pathname , '<STR_LIT>' ) <EOL> if local_path : <EOL> patterns . append ( local_path ) <EOL> if not obj . binary : <EOL> textfiles . append ( local_path ) <EOL> if self . stdin and self . stdin != self . DEV_NULL : <EOL> patterns . append ( self . stdin ) <EOL> textfiles . append ( self . stdin ) <EOL> if patterns : <EOL> self . _send_inputs ( patterns , textfiles ) <EOL> else : <EOL> self . _logger . debug ( '<STR_LIT>' ) <EOL> self . _logger . info ( '<STR_LIT>' , self . command ) <EOL> start_time = time . time ( ) <EOL> return_code , error_msg = self . _server . execute_command ( rdesc ) <EOL> et = time . time ( ) - start_time <EOL> if et >= <NUM_LIT> : <EOL> self . _logger . info ( '<STR_LIT>' , et ) <EOL> patterns = [ ] <EOL> textfiles = [ ] <EOL> for metadata in self . external_files : <EOL> if metadata . get ( '<STR_LIT>' , False ) : <EOL> patterns . append ( metadata . path ) <EOL> if not metadata . binary : <EOL> textfiles . append ( metadata . path ) <EOL> for pathname , obj in self . items ( iotype = '<STR_LIT>' , recurse = True ) : <EOL> if isinstance ( obj , FileRef ) : <EOL> patterns . append ( obj . path ) <EOL> if not obj . binary : <EOL> textfiles . append ( obj . path ) <EOL> patterns . append ( rdesc [ '<STR_LIT>' ] ) <EOL> textfiles . append ( rdesc [ '<STR_LIT>' ] ) <EOL> if self . stderr != self . STDOUT : <EOL> patterns . append ( rdesc [ '<STR_LIT>' ] ) <EOL> textfiles . append ( rdesc [ '<STR_LIT>' ] ) <EOL> self . _retrieve_results ( patterns , textfiles ) <EOL> if not self . stdout : <EOL> name = rdesc [ '<STR_LIT>' ] <EOL> if os . path . exists ( name ) : <EOL> with open ( name , '<STR_LIT>' ) as inp : <EOL> sys . stdout . write ( inp . read ( ) ) <EOL> os . remove ( name ) <EOL> else : <EOL> sys . stdout . write ( '<STR_LIT>' ) <EOL> if not self . stderr : <EOL> name = rdesc [ '<STR_LIT>' ] <EOL> if os . path . exists ( name ) : <EOL> with open ( name , '<STR_LIT>' ) as inp : <EOL> sys . stderr . write ( inp . read ( ) ) <EOL> os . remove ( name ) <EOL> else : <EOL> sys . stdout . write ( '<STR_LIT>' ) <EOL> finally : <EOL> RAM . release ( self . _server ) <EOL> self . _server = None <EOL> return ( return_code , error_msg ) <EOL> def _send_inputs ( self , patterns , textfiles ) : <EOL> """<STR_LIT>""" <EOL> self . _logger . info ( '<STR_LIT>' ) <EOL> start_time = time . time ( ) <EOL> filename = '<STR_LIT>' <EOL> pfiles , pbytes = pack_zipfile ( patterns , filename , self . _logger ) <EOL> try : <EOL> filexfer ( None , filename , self . _server , filename , '<STR_LIT:b>' , False ) <EOL> ufiles , ubytes = self . _server . unpack_zipfile ( filename , <EOL> textfiles = textfiles ) <EOL> finally : <EOL> os . remove ( filename ) <EOL> self . _server . remove ( filename ) <EOL> if ufiles != pfiles or ubytes != pbytes : <EOL> msg = '<STR_LIT>' % ( ufiles , ubytes , pfiles , pbytes ) <EOL> self . raise_exception ( msg , RuntimeError ) <EOL> et = time . time ( ) - start_time <EOL> if et >= <NUM_LIT> : <EOL> self . _logger . info ( '<STR_LIT>' , et ) <EOL> def _retrieve_results ( self , patterns , textfiles ) : <EOL> """<STR_LIT>""" <EOL> self . _logger . info ( '<STR_LIT>' ) <EOL> start_time = time . time ( ) <EOL> filename = '<STR_LIT>' <EOL> pfiles , pbytes = self . _server . pack_zipfile ( patterns , filename ) <EOL> filexfer ( self . _server , filename , None , filename , '<STR_LIT:b>' , False ) <EOL> try : <EOL> if os . path . getsize ( filename ) > <NUM_LIT:0> : <EOL> ufiles , ubytes = unpack_zipfile ( filename , logger = self . _logger , <EOL> textfiles = textfiles ) <EOL> else : <EOL> ufiles , ubytes = <NUM_LIT:0> , <NUM_LIT:0> <EOL> finally : <EOL> os . remove ( filename ) <EOL> self . _server . remove ( filename ) <EOL> if ufiles != pfiles or ubytes != pbytes : <EOL> msg = '<STR_LIT>' % ( ufiles , ubytes , pfiles , pbytes ) <EOL> self . raise_exception ( msg , RuntimeError ) <EOL> et = time . time ( ) - start_time <EOL> if et >= <NUM_LIT> : <EOL> self . _logger . info ( '<STR_LIT>' , et ) <EOL> def stop ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _stop = True <EOL> if self . _process : <EOL> self . _process . terminate ( ) <EOL> def copy_inputs ( self , inputs_dir , patterns ) : <EOL> """<STR_LIT>""" <EOL> self . _logger . info ( '<STR_LIT>' , inputs_dir ) <EOL> with self . dir_context : <EOL> if not os . path . exists ( inputs_dir ) : <EOL> self . raise_exception ( "<STR_LIT>" % inputs_dir , RuntimeError ) <EOL> self . _copy ( inputs_dir , patterns ) <EOL> def copy_results ( self , results_dir , patterns ) : <EOL> """<STR_LIT>""" <EOL> self . _logger . info ( '<STR_LIT>' , results_dir ) <EOL> with self . dir_context : <EOL> if not os . path . exists ( results_dir ) : <EOL> self . raise_exception ( "<STR_LIT>" % results_dir , RuntimeError ) <EOL> self . _copy ( results_dir , patterns ) <EOL> def _copy ( self , directory , patterns ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( patterns , basestring ) : <EOL> patterns = [ patterns ] <EOL> for pattern in patterns : <EOL> pattern = os . path . join ( directory , pattern ) <EOL> for src_path in sorted ( glob . glob ( pattern ) ) : <EOL> dst_path = os . path . basename ( src_path ) <EOL> self . _logger . debug ( '<STR_LIT>' , src_path ) <EOL> shutil . copy ( src_path , dst_path ) <EOL> mode = os . stat ( dst_path ) . st_mode <EOL> mode |= stat . S_IWUSR <EOL> os . chmod ( dst_path , mode ) <EOL> class _AccessController ( AccessController ) : <EOL> """<STR_LIT>""" <EOL> def check_access ( self , role , methodname , obj , attr ) : <EOL> """<STR_LIT>""" <EOL> if attr in ( '<STR_LIT>' , '<STR_LIT>' ) and methodname == '<STR_LIT>' : <EOL> raise RoleError ( '<STR_LIT>' % ( methodname , attr ) ) </s>
