<s> import os <EOL> import pickle <EOL> import inspect <EOL> from pyleaf . log import send as dbgstr <EOL> from pyleaf . rrc import resource <EOL> import copy <EOL> import inspect <EOL> import time <EOL> import sys <EOL> class protocol ( ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , graph , mods , folder , doc ) : <EOL> untrustme1 = list ( ) <EOL> untrustme2 = list ( ) <EOL> dbgstr ( '<STR_LIT>' + folder ) <EOL> self . _metafolder = folder <EOL> self . _rootdir = os . getcwd ( ) <EOL> self . _resmap = dict ( ) <EOL> self . _doc = doc <EOL> self . _graphres = resource ( '<STR_LIT>' , os . path . join ( folder , '<STR_LIT>' ) ) <EOL> untrustme1 = self . _updateGraph ( graph ) <EOL> for res in self . _getResNames ( ) : <EOL> newres = resource ( res , folder + '<STR_LIT:/>' + res + '<STR_LIT>' ) <EOL> self . _addResource ( res , newres ) <EOL> for mod in self . _getNodeNames ( ) : <EOL> newres = resource ( mod , folder + '<STR_LIT:/>' + mod + '<STR_LIT>' ) <EOL> self . _modules [ mod ] = newres <EOL> untrustme2 = self . _updateModules ( mods ) <EOL> for mod in untrustme1 : <EOL> self . untrust ( mod ) <EOL> for mod in untrustme2 : <EOL> self . untrust ( mod ) <EOL> def _updateGraph ( self , graph ) : <EOL> self . _graphres . setValue ( graph ) <EOL> untrustme1 = list ( ) <EOL> if self . _graphres . changed ( ) : <EOL> if self . _graphres . isDumped ( ) : <EOL> untrustme1 = self . _manageGraphChange ( self . _graphres . getFingerprint ( ) ) <EOL> self . _graphres . update ( ) <EOL> return untrustme1 <EOL> def _update ( self , graph , mods ) : <EOL> untrustme1 = list ( ) <EOL> untrustme2 = list ( ) <EOL> dbgstr ( '<STR_LIT>' + str ( graph ) , <NUM_LIT:3> ) <EOL> dbgstr ( '<STR_LIT>' + str ( mods ) , <NUM_LIT:3> ) <EOL> untrustme1 = self . _updateGraph ( graph ) <EOL> untrustme2 = self . _updateModules ( mods ) <EOL> for mod in untrustme1 : <EOL> self . untrust ( mod ) <EOL> for mod in untrustme2 : <EOL> self . untrust ( mod ) <EOL> def listinputs ( self , node ) : <EOL> """<STR_LIT>""" <EOL> return self . _getGraph ( ) . getInNodes ( node ) <EOL> def listoutputs ( self , node ) : <EOL> """<STR_LIT>""" <EOL> return self . _getGraph ( ) . getOutNodes ( node ) <EOL> def resetall ( self ) : <EOL> """<STR_LIT>""" <EOL> for res in self . _getResNames ( ) : <EOL> self . reset ( res ) <EOL> def clearall ( self ) : <EOL> """<STR_LIT>""" <EOL> for res in self . _getResNames ( ) : <EOL> self . clear ( res ) <EOL> def undumpall ( self ) : <EOL> """<STR_LIT>""" <EOL> for res in self . _getResNames ( ) : <EOL> self . undump ( res ) <EOL> if os . path . exists ( self . _metafolder ) : <EOL> if os . listdir ( self . _metafolder ) == [ ] : <EOL> os . rmdir ( self . _metafolder ) <EOL> def getinputs ( self , mod ) : <EOL> """<STR_LIT>""" <EOL> if type ( mod ) != str : <EOL> mod = mod . __name__ <EOL> innodes = self . _getInNodes ( mod ) <EOL> ids = [ self . _getGraph ( ) . getAttrib ( _node , '<STR_LIT:id>' ) for _node in innodes ] <EOL> nodeparams = [ copy . deepcopy ( self . _getResource ( innode ) . getValue ( ) ) for innode in innodes ] <EOL> nodeparams = [ one for ( one , two ) in sorted ( zip ( nodeparams , ids ) , key = lambda x : x [ <NUM_LIT:1> ] ) ] <EOL> return tuple ( nodeparams ) <EOL> def run ( self , parallel = False ) : <EOL> """<STR_LIT>""" <EOL> res = None <EOL> allok = True <EOL> l = self . _getLeaves ( ) <EOL> for leaf in l : <EOL> resname = leaf <EOL> if not self . _isAvailable ( resname ) and not self . _isDumped ( resname ) : <EOL> allok = False <EOL> if allok : <EOL> dbgstr ( '<STR_LIT>' ) <EOL> else : <EOL> res = self . provide ( l , parallel ) <EOL> return res <EOL> def untrust ( self , nodename ) : <EOL> """<STR_LIT>""" <EOL> if type ( nodename ) != str : <EOL> nodename = nodename . __name__ <EOL> self . reset ( nodename ) <EOL> dependents = self . _getOutNodesRecursive ( nodename ) <EOL> for res in self . _getResNames ( ) : <EOL> if ( res in dependents ) : <EOL> self . reset ( res ) <EOL> def clear ( self , filtername , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> if type ( filtername ) != str : <EOL> filtername = filtername . __name__ <EOL> if self . _isAvailable ( filtername ) : <EOL> if verbose : <EOL> dbgstr ( '<STR_LIT>' + str ( filtername ) ) <EOL> self . _resmap [ filtername ] . clear ( ) <EOL> def reset ( self , filtername ) : <EOL> """<STR_LIT>""" <EOL> if self . _isAvailable ( filtername ) or self . _isDumped ( filtername ) : <EOL> dbgstr ( '<STR_LIT>' + str ( filtername ) ) <EOL> self . clear ( filtername , False ) <EOL> self . undump ( filtername , False ) <EOL> def undump ( self , filtername , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> if type ( filtername ) != str : <EOL> filtername = filtername . __name__ <EOL> if self . _isDumped ( filtername ) : <EOL> if verbose : <EOL> dbgstr ( '<STR_LIT>' + str ( filtername ) ) <EOL> self . _getResource ( filtername ) . clearDump ( ) <EOL> def provideSerial ( self , resname ) : <EOL> """<STR_LIT>""" <EOL> if type ( resname ) != str : <EOL> resname = resname . __name__ <EOL> return self . _provideResource ( resname ) . getValue ( ) <EOL> def rebuild ( self , resname ) : <EOL> """<STR_LIT>""" <EOL> self . clear ( resname ) <EOL> self . provide ( resname ) <EOL> def dumpOn ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _dodump = True <EOL> def dumpOff ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _dodump = False <EOL> def trust ( self , who , what ) : <EOL> """<STR_LIT>""" <EOL> if type ( who ) != str : <EOL> who = who . __name__ <EOL> if type ( who ) == str : <EOL> self . _setMod ( who , what ) <EOL> dbgstr ( '<STR_LIT>' + str ( what ) + '<STR_LIT>' + who + '<STR_LIT>' , <NUM_LIT:0> ) <EOL> elif type ( who ) == tuple : <EOL> self . _addResource ( who , what ) <EOL> dbgstr ( '<STR_LIT>' + str ( type ( what ) ) + '<STR_LIT>' + str ( who ) + '<STR_LIT>' , <NUM_LIT:0> ) <EOL> def list ( self ) : <EOL> """<STR_LIT>""" <EOL> mystr = '<STR_LIT>' <EOL> for res in self . _getResNames ( ) : <EOL> mystr += str ( res ) + '<STR_LIT>' <EOL> if self . _isAvailable ( res ) : <EOL> mystr += '<STR_LIT>' <EOL> else : <EOL> mystr += '<STR_LIT>' <EOL> if self . _isDumped ( res ) : <EOL> mystr += '<STR_LIT>' <EOL> else : <EOL> mystr += '<STR_LIT>' <EOL> mystr += '<STR_LIT:\n>' <EOL> print ( mystr ) <EOL> def export ( self , ofile , layout = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> import textwrap <EOL> ofile = ofile <EOL> f = open ( ofile , '<STR_LIT:w>' ) <EOL> f . write ( '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> ( '<STR_LIT>' if layout . lower ( ) == '<STR_LIT>' else '<STR_LIT>' ) + <EOL> '<STR_LIT:;>' ) <EOL> for idx , node in enumerate ( self . _getGraph ( ) . getNodes ( ) ) : <EOL> if self . _isFileMod ( node ) : <EOL> shape = '<STR_LIT>' <EOL> else : <EOL> shape = '<STR_LIT>' <EOL> f . write ( str ( node ) ) <EOL> docstr = inspect . getdoc ( self . _modules [ node ] . getValue ( ) ) if type ( self . _modules [ node ] . getValue ( ) ) == type ( inspect . getdoc ) else None <EOL> f . write ( '<STR_LIT>' + shape + '<STR_LIT>' + <EOL> self . _getGraph ( ) . getAttrib ( node , '<STR_LIT:label>' ) + <EOL> '<STR_LIT>' + <EOL> ( '<STR_LIT:->' if docstr == None else textwrap . fill ( docstr , <NUM_LIT:30> ) ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) + <EOL> '<STR_LIT>' ) <EOL> for node in self . _getGraph ( ) . keys ( ) : <EOL> for onode in self . _getGraph ( ) [ node ] : <EOL> f . write ( node + '<STR_LIT>' + onode + '<STR_LIT:\n>' ) <EOL> f . write ( '<STR_LIT:}>' ) <EOL> f . close ( ) <EOL> t = os . system ( '<STR_LIT>' + ofile + '<STR_LIT>' + ofile ) <EOL> if t != <NUM_LIT:0> : <EOL> raise NameError ( '<STR_LIT>' ) <EOL> def _readabletime ( self , secs ) : <EOL> mins , secs = divmod ( secs , <NUM_LIT> ) <EOL> hours , mins = divmod ( mins , <NUM_LIT> ) <EOL> return '<STR_LIT>' % ( hours , mins , secs ) <EOL> def publish ( self , ofile , odir = '<STR_LIT>' , layout = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> import textwrap <EOL> import os . path <EOL> oname = ofile <EOL> if odir == '<STR_LIT>' : <EOL> odir = '<STR_LIT:html>' <EOL> if not os . path . exists ( odir ) : <EOL> os . mkdir ( odir ) <EOL> fullname = os . path . join ( odir , ofile ) <EOL> f = open ( fullname + '<STR_LIT>' , '<STR_LIT:w>' ) <EOL> f . write ( '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> ( '<STR_LIT>' if layout . lower ( ) == '<STR_LIT>' else '<STR_LIT>' ) + <EOL> '<STR_LIT>' ) <EOL> for idx , node in enumerate ( self . _getGraph ( ) . getNodes ( ) ) : <EOL> f . write ( str ( node ) ) <EOL> docstr = inspect . getdoc ( self . _modules [ node ] . getValue ( ) ) if type ( self . _modules [ node ] . getValue ( ) ) == type ( inspect . getdoc ) else None <EOL> if self . _isFileMod ( node ) : <EOL> shape = '<STR_LIT>' <EOL> if self . _isAvailable ( node ) : <EOL> content = self . _getResource ( node ) . getValue ( ) <EOL> if isinstance ( content , tuple ) or isinstance ( content , list ) : <EOL> output_line = '<STR_LIT>' + '<STR_LIT>' + '<STR_LIT>' <EOL> else : <EOL> output_line = '<STR_LIT>' + content + '<STR_LIT>' <EOL> else : <EOL> output_line = '<STR_LIT>' <EOL> else : <EOL> output_line = '<STR_LIT>' <EOL> shape = '<STR_LIT>' <EOL> f . write ( '<STR_LIT>' + shape + '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> self . _getGraph ( ) . getAttrib ( node , '<STR_LIT:label>' ) + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> ( '<STR_LIT:->' if docstr == None else textwrap . fill ( docstr , <NUM_LIT:30> ) ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' + node + '<STR_LIT>' + <EOL> output_line + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' ) <EOL> for node in self . _getGraph ( ) . keys ( ) : <EOL> for onode in self . _getGraph ( ) [ node ] : <EOL> f . write ( node + '<STR_LIT>' + onode + '<STR_LIT:\n>' ) <EOL> f . write ( '<STR_LIT:}>' ) <EOL> f . close ( ) <EOL> t = os . system ( '<STR_LIT>' + <EOL> fullname + '<STR_LIT>' + fullname + '<STR_LIT>' + fullname + '<STR_LIT>' ) <EOL> t = os . system ( '<STR_LIT>' + fullname + '<STR_LIT>' + fullname + '<STR_LIT>' ) <EOL> if t != <NUM_LIT:0> : <EOL> raise NameError ( '<STR_LIT>' ) <EOL> html_header = """<STR_LIT>""" + oname + """<STR_LIT>""" <EOL> html_footer = """<STR_LIT>""" <EOL> f = open ( fullname + '<STR_LIT>' , '<STR_LIT:w>' ) <EOL> f . write ( html_header ) <EOL> f . write ( '<STR_LIT>' + oname + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> if len ( self . _doc ) > <NUM_LIT:0> : <EOL> f . write ( '<STR_LIT>' + self . _doc + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' + str ( len ( self . _getGraph ( ) ) ) + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' + str ( sum ( [ self . _isFileMod ( node ) for node in self . _getGraph ( ) . getNodes ( ) ] ) ) + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' + self . _readabletime ( sum ( [ self . _resmap [ x ] . _buildtime for x in self . _resmap ] ) ) + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' + oname + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> map = open ( fullname + '<STR_LIT>' ) . read ( ) <EOL> f . write ( map ) <EOL> f . write ( '<STR_LIT>' '<STR_LIT>' ) <EOL> for idx , node in enumerate ( self . _getGraph ( ) . getNodes ( ) ) : <EOL> f . write ( '<STR_LIT>' + str ( node ) + '<STR_LIT>' + str ( node ) + '<STR_LIT>' ) <EOL> docstr = inspect . getdoc ( self . _modules [ node ] . getValue ( ) ) if type ( self . _modules [ node ] . getValue ( ) ) == type ( inspect . getdoc ) else None <EOL> if docstr == None : <EOL> f . write ( '<STR_LIT>' ) <EOL> else : <EOL> f . write ( docstr ) <EOL> if not self . _resmap [ node ] . _timestamp == None : <EOL> tstamp = self . _resmap [ node ] . _timestamp <EOL> btime = self . _resmap [ node ] . _buildtime <EOL> else : <EOL> tstamp = '<STR_LIT>' <EOL> btime = '<STR_LIT>' <EOL> f . write ( '<STR_LIT>' + tstamp + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' + str ( self . _readabletime ( btime ) ) + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> srccode = inspect . getsource ( self . _modules [ node ] . getValue ( ) ) if type ( self . _modules [ node ] . getValue ( ) ) == type ( inspect . getdoc ) else None <EOL> f . write ( srccode . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' + time . asctime ( ) + '<STR_LIT>' ) <EOL> f . write ( '<STR_LIT>' ) <EOL> f . write ( html_footer ) <EOL> f . close ( ) <EOL> try : <EOL> from pkg_resources import resource_filename <EOL> stylef = resource_filename ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> logof = resource_filename ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> import shutil <EOL> shutil . copy ( stylef , odir ) <EOL> shutil . copy ( logof , odir ) <EOL> except : <EOL> print ( '<STR_LIT>' ) <EOL> def _prettyPrint ( self , x ) : <EOL> return str ( x ) . strip ( '<STR_LIT>' ) . replace ( "<STR_LIT:'>" , "<STR_LIT>" ) <EOL> def provide ( self , res , parallel = False ) : <EOL> t = time . time ( ) <EOL> if not type ( res ) == list : <EOL> temp = list ( ) <EOL> temp . append ( res ) <EOL> res = temp <EOL> for i , r in enumerate ( res ) : <EOL> if type ( res [ i ] ) != str : <EOL> res [ i ] = res [ i ] . __name__ <EOL> D = self . _findDependancies ( res ) <EOL> D . update ( res ) <EOL> states = self . _getBestStates ( D ) <EOL> if len ( states [ <NUM_LIT:1> ] ) > <NUM_LIT:0> : <EOL> dbgstr ( '<STR_LIT>' + <EOL> self . _prettyPrint ( states [ <NUM_LIT:1> ] ) ) <EOL> for node in states [ <NUM_LIT:1> ] : <EOL> self . _provideResource ( node ) <EOL> if len ( states [ <NUM_LIT:2> ] ) > <NUM_LIT:0> : <EOL> dbgstr ( '<STR_LIT>' + <EOL> self . _prettyPrint ( states [ <NUM_LIT:2> ] ) ) <EOL> needbuild = states [ <NUM_LIT:2> ] <EOL> while True : <EOL> canrun = self . _findRunnable ( needbuild ) <EOL> if len ( canrun ) > <NUM_LIT:1> and parallel : <EOL> dbgstr ( '<STR_LIT>' + <EOL> self . _prettyPrint ( canrun ) ) <EOL> self . _runNodes ( canrun , parallel ) <EOL> needbuild = [ x for x in needbuild if not x in canrun ] <EOL> if len ( canrun ) == <NUM_LIT:0> : <EOL> break <EOL> builtres = list ( ) <EOL> for resname in res : <EOL> builtres . append ( self . _resmap [ resname ] . getValue ( ) ) <EOL> if len ( states [ <NUM_LIT:1> ] ) > <NUM_LIT:0> or len ( states [ <NUM_LIT:2> ] ) > <NUM_LIT:0> : <EOL> dbgstr ( '<STR_LIT>' + self . _readabletime ( time . time ( ) - t ) + '<STR_LIT:.>' ) <EOL> if len ( builtres ) > <NUM_LIT:1> : <EOL> return builtres <EOL> else : <EOL> return builtres [ <NUM_LIT:0> ] <EOL> def _runNodes ( self , nodes , parallel = False ) : <EOL> if parallel and len ( nodes ) > <NUM_LIT:1> : <EOL> from multiprocessing import Process , Queue <EOL> results = Queue ( ) <EOL> tasks = list ( ) <EOL> for node in nodes : <EOL> nodeparams = self . _getNodePar ( node ) <EOL> tasks . append ( <EOL> Process ( target = self . _callModPar , args = ( node , nodeparams , results , ) ) <EOL> ) <EOL> for task in tasks : task . start ( ) <EOL> for task in tasks : task . join ( ) <EOL> for task in tasks : <EOL> taskres = results . get ( ) <EOL> resname = self . _buildResName ( taskres [ <NUM_LIT:0> ] , None , taskres [ <NUM_LIT:1> ] ) <EOL> dbgstr ( '<STR_LIT>' + taskres [ <NUM_LIT:0> ] , <NUM_LIT:2> ) <EOL> t = taskres [ <NUM_LIT:2> ] <EOL> self . _newResource ( resname , taskres [ <NUM_LIT:1> ] , t ) <EOL> else : <EOL> for node in nodes : <EOL> nodeparams = self . _getNodePar ( node ) <EOL> taskres = self . _callMod ( node , nodeparams ) <EOL> resname = self . _buildResName ( taskres [ <NUM_LIT:1> ] , None , taskres [ <NUM_LIT:0> ] ) <EOL> dbgstr ( '<STR_LIT>' + resname , <NUM_LIT:2> ) <EOL> t = taskres [ <NUM_LIT:2> ] <EOL> self . _newResource ( resname , taskres [ <NUM_LIT:0> ] , t ) <EOL> def time ( self , node ) : <EOL> if type ( node ) != str : <EOL> node = node . __name__ <EOL> return ( self . _resmap [ node ] . _timestamp , <EOL> self . _resmap [ node ] . _buildtime ) <EOL> def _findRunnable ( self , nodes ) : <EOL> runnable = list ( ) <EOL> for node in nodes : <EOL> if all ( [ self . _isAvailable ( n ) for n in self . _getInNodes ( node ) ] ) : <EOL> runnable . append ( node ) <EOL> return runnable <EOL> def _getBestStates ( self , D ) : <EOL> ar = list ( ) <EOL> dr = list ( ) <EOL> nr = list ( ) <EOL> for d in D : <EOL> if self . _isAvailable ( d ) : <EOL> ar . append ( d ) <EOL> elif self . _isDumped ( d ) : <EOL> dr . append ( d ) <EOL> else : <EOL> nr . append ( d ) <EOL> return ar , dr , nr <EOL> def _findDependancies ( self , nodes ) : <EOL> nec = set ( ) <EOL> for n in nodes : <EOL> nec . update ( self . _getGraph ( ) . getAncestors ( n ) ) <EOL> return nec <EOL> def _updateModules ( self , mods ) : <EOL> untrustme = list ( ) <EOL> for modname in mods : <EOL> if modname in self . _modules . keys ( ) : <EOL> self . _modules [ modname ] . setValue ( mods [ modname ] ) <EOL> if self . _modules [ modname ] . changed ( ) : <EOL> untrustme . append ( modname ) <EOL> self . _modhelp [ modname ] = inspect . getdoc ( self . _modules [ modname ] . getValue ( ) ) <EOL> self . _modules [ modname ] . update ( ) <EOL> else : <EOL> dbgstr ( '<STR_LIT>' + modname ) <EOL> self . _modules [ modname ] = resource ( modname , <EOL> self . _metafolder + '<STR_LIT:/>' + modname + '<STR_LIT>' ) <EOL> self . _addResource ( modname , self . _modules [ modname ] ) <EOL> self . _modules [ modname ] . setValue ( mods [ modname ] ) <EOL> self . _modules [ modname ] . update ( ) <EOL> return untrustme <EOL> def _manageGraphChange ( self , newGraph ) : <EOL> if newGraph == None : <EOL> return <EOL> oldg = self . _graphres . getValue ( ) <EOL> untrustme = list ( ) <EOL> for node in newGraph . getNodes ( ) : <EOL> if node in oldg . getNodes ( ) : <EOL> innodes1 = newGraph . getInNodes ( node ) <EOL> innodes2 = oldg . getInNodes ( node ) <EOL> for inNode in innodes1 : <EOL> if not inNode in innodes2 : <EOL> dbgstr ( '<STR_LIT>' + node + '<STR_LIT>' ) <EOL> untrustme . append ( node ) <EOL> return untrustme <EOL> def _getContents ( self , mod ) : <EOL> try : <EOL> inspect . getsource ( mod ) <EOL> value = inspect . getsource ( mod ) <EOL> except Exception : <EOL> value = mod <EOL> return value <EOL> def _setMod ( self , modname , mod ) : <EOL> haschanged = self . _modules [ modname ] . changed ( ) <EOL> if haschanged : <EOL> if type ( mod ) == File : <EOL> mod . update ( ) <EOL> self . _modules [ modname ] . setValue ( mod . getValue ( ) ) <EOL> self . _modules [ modname ] . updateFingerprint ( ) <EOL> self . untrust ( modname ) <EOL> dependents = self . _getOutNodesRecursive ( modname ) <EOL> if dependents != [ ] : <EOL> dbgstr ( '<STR_LIT>' + str ( dependents ) , <NUM_LIT:0> ) <EOL> def _setMods ( self , mods ) : <EOL> self . _modules = mods <EOL> for modname in mods . keys ( ) : <EOL> self . _setMod ( modname , mods [ modname ] ) <EOL> dbgstr ( '<STR_LIT>' + str ( self . _modules ) , <NUM_LIT:3> ) <EOL> def _newResource ( self , resname , resval , t ) : <EOL> dbgstr ( '<STR_LIT>' + resname , <NUM_LIT:2> ) <EOL> dbgstr ( '<STR_LIT>' + str ( resval ) , <NUM_LIT:3> ) <EOL> self . _getResource ( resname ) . setValue ( resval ) <EOL> self . _getResource ( resname ) . updateFingerprint ( ) <EOL> self . _getResource ( resname ) . _buildtime = t <EOL> self . _getResource ( resname ) . _timestamp = time . asctime ( ) <EOL> dbgstr ( '<STR_LIT>' + resname ) <EOL> self . _getResource ( resname ) . dump ( ) <EOL> def _clearFilter ( self , nodename ) : <EOL> if self . _isLeaf ( nodename ) : <EOL> self . clear ( nodename ) <EOL> for node in self . getoutputs ( nodename ) : <EOL> self . clear ( node ) <EOL> def _setWdir ( self , wdir ) : <EOL> self . wdir = wdir <EOL> def _isAvailable ( self , resname ) : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) , <NUM_LIT:3> ) <EOL> if self . _getResource ( resname ) . isAvailable ( ) : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) , <NUM_LIT:3> ) <EOL> return True <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) , <NUM_LIT:3> ) <EOL> return False <EOL> def _getResource ( self , resname ) : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) , <NUM_LIT:3> ) <EOL> return self . _resmap [ resname ] <EOL> def _addResource ( self , name , res ) : <EOL> self . _resmap [ name ] = res <EOL> def _getNodeNames ( self ) : <EOL> return self . _getGraph ( ) . getNodes ( ) <EOL> def _isResFile ( self , res ) : <EOL> isit = res . isFile ( ) <EOL> if isit : dbgstr ( str ( res ) + '<STR_LIT>' ) <EOL> return isit <EOL> def _provideResource ( self , resname ) : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) , <NUM_LIT:2> ) <EOL> if self . _isAvailable ( resname ) : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) ) <EOL> dbgstr ( '<STR_LIT>' + str ( self . _getResource ( resname ) ) , <NUM_LIT:4> ) <EOL> return self . _getResource ( resname ) <EOL> elif self . _isDumped ( resname ) : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) , <NUM_LIT:2> ) <EOL> self . _addResource ( resname , self . _loadResource ( resname ) ) <EOL> dbgstr ( '<STR_LIT>' + str ( self . _getResource ( resname ) ) , <NUM_LIT:4> ) <EOL> return self . _resmap [ resname ] <EOL> else : <EOL> dbgstr ( '<STR_LIT>' + str ( resname ) ) <EOL> self . _runNode ( resname ) <EOL> return self . _getResource ( resname ) <EOL> def _runNode ( self , node ) : <EOL> dbgstr ( '<STR_LIT>' + str ( node ) ) <EOL> nodeparams = list ( ) <EOL> input_nodes = self . _getInNodes ( node ) <EOL> for item in input_nodes : <EOL> thisnode_inputs = list ( ) <EOL> neededres = item <EOL> dbgstr ( '<STR_LIT>' + neededres ) <EOL> this_params = self . _provideResource ( neededres ) <EOL> if type ( this_params . getValue ( ) ) == list : <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> for this_param in this_params . getValue ( ) : <EOL> thisnode_inputs . append ( this_param ) <EOL> elif self . _isResFile ( this_params ) : <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> thisnode_inputs . append ( this_params . getValue ( ) ) <EOL> else : <EOL> dbgstr ( '<STR_LIT>' + str ( type ( this_params . getValue ( ) ) ) , <NUM_LIT:2> ) <EOL> thisnode_inputs . append ( this_params . getValue ( ) ) <EOL> if len ( thisnode_inputs ) == <NUM_LIT:1> : <EOL> nodeparams . append ( thisnode_inputs [ <NUM_LIT:0> ] ) <EOL> else : <EOL> nodeparams . append ( thisnode_inputs ) <EOL> dbgstr ( '<STR_LIT>' + node , <NUM_LIT:2> ) <EOL> dbgstr ( '<STR_LIT>' + str ( self . _getModule ( node ) . getValue ( ) ) , <NUM_LIT:2> ) <EOL> dbgstr ( '<STR_LIT>' + str ( nodeparams ) , <NUM_LIT:3> ) <EOL> newres , newresname = self . _callMod ( node , nodeparams ) <EOL> dbgstr ( '<STR_LIT>' + str ( self . _resmap [ newresname ] . _buildtime ) ) <EOL> return newres <EOL> def _getNodePar ( self , node ) : <EOL> nodeparams = list ( ) <EOL> input_nodes = self . _getInNodes ( node ) <EOL> for item in input_nodes : <EOL> thisnode_inputs = list ( ) <EOL> neededres = item <EOL> this_params = self . _getResource ( neededres ) <EOL> if type ( this_params . getValue ( ) ) == list : <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> for this_param in this_params . getValue ( ) : <EOL> thisnode_inputs . append ( this_param ) <EOL> elif self . _isResFile ( this_params ) : <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> thisnode_inputs . append ( this_params . getValue ( ) ) <EOL> else : <EOL> dbgstr ( '<STR_LIT>' + str ( type ( this_params . getValue ( ) ) ) , <NUM_LIT:2> ) <EOL> thisnode_inputs . append ( this_params . getValue ( ) ) <EOL> if len ( thisnode_inputs ) == <NUM_LIT:1> : <EOL> nodeparams . append ( thisnode_inputs [ <NUM_LIT:0> ] ) <EOL> else : <EOL> nodeparams . append ( thisnode_inputs ) <EOL> return nodeparams <EOL> def _getModule ( self , name ) : <EOL> return self . _modules [ name ] <EOL> def _callMod ( self , node , nodeparams ) : <EOL> t = time . time ( ) <EOL> if not self . _checkIsFunction ( self . _modules [ node ] . getValue ( ) ) : <EOL> dbgstr ( '<STR_LIT>' + node + '<STR_LIT>' , <NUM_LIT:2> ) <EOL> newres = self . _modules [ node ] . getValue ( ) <EOL> elif len ( nodeparams ) == <NUM_LIT:0> : <EOL> dbgstr ( '<STR_LIT>' + str ( node ) , <NUM_LIT:2> ) <EOL> dbgstr ( '<STR_LIT>' + node ) <EOL> newres = self . _modules [ node ] . getValue ( ) ( ) <EOL> dbgstr ( '<STR_LIT>' ) <EOL> dbgstr ( '<STR_LIT>' + str ( newres ) , <NUM_LIT:3> ) <EOL> elif self . _getGraph ( ) . getAttrib ( node , '<STR_LIT>' ) : <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> dbgstr ( '<STR_LIT>' + node ) <EOL> newres = self . _modules [ node ] . getValue ( ) ( * nodeparams ) <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> dbgstr ( '<STR_LIT>' + str ( newres ) , <NUM_LIT:3> ) <EOL> else : <EOL> dbgstr ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> for nodeparam in nodeparams : <EOL> dbgstr ( '<STR_LIT>' + node ) <EOL> newres = self . _modules [ node ] . getValue ( ) ( nodeparam ) <EOL> dbgstr ( '<STR_LIT>' ) <EOL> dbgstr ( '<STR_LIT>' + str ( newres ) , <NUM_LIT:3> ) <EOL> newresname = self . _buildResName ( node , None , newres ) <EOL> dbgstr ( '<STR_LIT>' + node , <NUM_LIT:2> ) <EOL> t = time . time ( ) - t <EOL> return newres , newresname , t <EOL> def _callModPar ( self , node , nodeparams , queue ) : <EOL> dbgstr ( '<STR_LIT>' + node ) <EOL> funct = self . _modules [ node ] . getValue ( ) <EOL> if len ( nodeparams ) == <NUM_LIT:0> : <EOL> dbgstr ( '<STR_LIT>' + str ( node ) , <NUM_LIT:2> ) <EOL> t = time . time ( ) <EOL> newres = funct ( ) <EOL> t = time . time ( ) - t <EOL> else : <EOL> t = time . time ( ) <EOL> newres = funct ( * nodeparams ) <EOL> t = time . time ( ) - t <EOL> dbgstr ( '<STR_LIT>' + str ( newres ) , <NUM_LIT:3> ) <EOL> queue . put ( ( node , newres , t ) ) <EOL> def _checkIsFunction ( self , x ) : <EOL> return type ( lambda y : y ) == type ( x ) <EOL> def _placeFileRes ( self , fname ) : <EOL> if self . _auto_place_files : <EOL> os . system ( '<STR_LIT>' + fname + '<STR_LIT>' + self . _metafolder ) <EOL> def _buildResName ( self , inode , onode , rawres ) : <EOL> if onode == None : <EOL> return inode <EOL> else : <EOL> return inode <EOL> def _isFileMod ( self , node ) : <EOL> flags = self . _getGraph ( ) . getAttrib ( node , '<STR_LIT>' ) <EOL> if flags == None : <EOL> return False <EOL> return '<STR_LIT:f>' in flags or '<STR_LIT:F>' in flags <EOL> def _updateFilePath ( self , path ) : <EOL> parts = os . path . split ( path ) <EOL> if parts [ <NUM_LIT:0> ] != self . _metafolder : <EOL> return self . _metafolder + '<STR_LIT:/>' + parts [ <NUM_LIT:1> ] <EOL> return path <EOL> def _processRawRes ( self , node , rawres ) : <EOL> newresname = self . _buildResName ( node , None , rawres ) <EOL> dbgstr ( '<STR_LIT>' + node , <NUM_LIT:2> ) <EOL> self . _newResource ( newresname , rawres ) <EOL> def _setDumpFolder ( self , f ) : <EOL> self . _metafolder = f <EOL> def _isDumped ( self , resname ) : <EOL> return self . _getResource ( resname ) . isDumped ( ) <EOL> def _loadResource ( self , res ) : <EOL> dbgstr ( '<STR_LIT>' + str ( res ) + '<STR_LIT>' , <NUM_LIT:2> ) <EOL> if self . _isDumped ( res ) : <EOL> dbgstr ( '<STR_LIT>' + str ( self . _resmap [ res ] ) + '<STR_LIT>' + self . _resmap [ res ] . getDumpPath ( ) , <NUM_LIT:2> ) <EOL> return pickle . load ( open ( self . _resmap [ res ] . getDumpPath ( ) , '<STR_LIT:rb>' ) ) <EOL> else : <EOL> dbgstr ( '<STR_LIT>' + str ( res ) + '<STR_LIT>' + self . _resmap [ res ] . getDumpPath ( ) ) <EOL> def _ChangeME_resToPath ( self , res ) : <EOL> if self . _getGraph ( ) . getAttrib ( res [ <NUM_LIT:0> ] , '<STR_LIT>' ) : <EOL> if res [ <NUM_LIT:0> ] == None : first_part = '<STR_LIT>' <EOL> else : first_part = res [ <NUM_LIT:0> ] <EOL> if res [ <NUM_LIT:1> ] == None : second_part = '<STR_LIT>' <EOL> else : second_part = res [ <NUM_LIT:1> ] <EOL> if first_part != '<STR_LIT>' and second_part != '<STR_LIT>' : <EOL> mid_part = '<STR_LIT>' <EOL> else : mid_part = '<STR_LIT>' <EOL> fname = first_part + mid_part + second_part + '<STR_LIT>' <EOL> else : <EOL> fname = res [ <NUM_LIT:0> ] <EOL> dbgstr ( '<STR_LIT>' + str ( res ) + '<STR_LIT>' + self . _metafolder + '<STR_LIT:/>' + fname , <NUM_LIT:2> ) <EOL> return self . _metafolder + '<STR_LIT:/>' + fname <EOL> def _getInNodes ( self , node ) : <EOL> return self . _getGraph ( ) . getInNodes ( node ) <EOL> def getoutputs ( self , node ) : <EOL> return self . _getGraph ( ) . getOutNodes ( node ) <EOL> def _isLeaf ( self , node ) : <EOL> return self . _getGraph ( ) [ node ] == [ ] <EOL> def _getOutNodesRecursive ( self , node ) : <EOL> alloutnodes = list ( ) <EOL> nodestack = list ( self . getoutputs ( node ) ) <EOL> while nodestack != [ ] : <EOL> onode = nodestack . pop ( ) <EOL> nodestack . extend ( self . getoutputs ( onode ) ) <EOL> alloutnodes . append ( onode ) <EOL> return alloutnodes <EOL> def _getLeaves ( self ) : <EOL> leaves = list ( ) <EOL> for key in self . _getGraph ( ) . keys ( ) : <EOL> if self . _getGraph ( ) [ key ] == [ ] : <EOL> leaves . append ( key ) <EOL> return leaves <EOL> def _getResNames ( self ) : <EOL> return self . _getGraph ( ) . getNodes ( ) <EOL> def _setDumping ( self , d ) : <EOL> self . _dodump = d <EOL> def _getGraph ( self ) : <EOL> return self . _graphres . getValue ( ) <EOL> def _setMod ( self , who , what ) : <EOL> self . _modcontents [ who . getName ( ) ] = self . _getContents ( what ) <EOL> self . dumpMods ( ) <EOL> def _loadMods ( self ) : <EOL> if os . path . exists ( self . modsToPath ( ) ) : <EOL> mods = pickle . load ( open ( self . modsToPath ( ) , '<STR_LIT:r>' ) ) <EOL> else : <EOL> mods = dict ( ) <EOL> dbgstr ( '<STR_LIT>' + str ( mods . keys ( ) ) , <NUM_LIT:2> ) <EOL> return mods <EOL> def _setMetaFolder ( self , f ) : <EOL> self . _metafolder = f <EOL> def _getresmap ( self ) : <EOL> return self . _resmap <EOL> _resmap = dict ( ) <EOL> _graphres = None <EOL> _metafolder = '<STR_LIT>' <EOL> _dodump = True <EOL> _modules = dict ( ) <EOL> _modhelp = dict ( ) <EOL> _auto_place_files = False <EOL> _doc = '<STR_LIT>' </s>
