<s> import cPickle , time , sys , os , traceback , types , json , signal <EOL> import numpy <EOL> from collections import OrderedDict <EOL> import theano . tensor as tt <EOL> import Mariana . network as MNET <EOL> import Mariana . training . recorders as MREC <EOL> import Mariana . training . stopcriteria as MSTOP <EOL> import Mariana . candies as MCAN <EOL> import Mariana . settings as MSET <EOL> __all__ = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> class Trainer_ABC ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . store = { <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : <NUM_LIT:0> , <EOL> } , <EOL> "<STR_LIT>" : { } , <EOL> "<STR_LIT>" : { } , <EOL> "<STR_LIT>" : { } , <EOL> } <EOL> def start ( self , runName , model , recorder , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> import simplejson , signal , cPickle <EOL> def _handler_sig_term ( sig , frame ) : <EOL> _dieGracefully ( "<STR_LIT>" , None ) <EOL> sys . exit ( sig ) <EOL> def _dieGracefully ( exType , tb = None ) : <EOL> if type ( exType ) is types . StringType : <EOL> exName = exType <EOL> else : <EOL> exName = exType . __name__ <EOL> death_time = time . ctime ( ) . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:_>' ) <EOL> filename = "<STR_LIT>" + runName + "<STR_LIT>" + exName + "<STR_LIT:_>" + death_time <EOL> sys . stderr . write ( "<STR_LIT>" % ( exName , filename ) ) <EOL> model . saveParameters ( filename ) <EOL> f = open ( filename + "<STR_LIT>" , '<STR_LIT:w>' ) <EOL> f . write ( "<STR_LIT>" ) <EOL> f . write ( "<STR_LIT>" ) <EOL> f . write ( "<STR_LIT>" % runName ) <EOL> f . write ( "<STR_LIT>" % os . getpid ( ) ) <EOL> f . write ( "<STR_LIT>" % str ( exType ) ) <EOL> f . write ( "<STR_LIT>" % death_time ) <EOL> f . write ( "<STR_LIT>" % filename ) <EOL> sstore = str ( self . store ) . replace ( "<STR_LIT:'>" , '<STR_LIT:">' ) . replace ( "<STR_LIT:True>" , '<STR_LIT:true>' ) . replace ( "<STR_LIT:False>" , '<STR_LIT:false>' ) <EOL> try : <EOL> f . write ( <EOL> "<STR_LIT>" % json . dumps ( <EOL> json . loads ( sstore ) , sort_keys = True , <EOL> indent = <NUM_LIT:4> , <EOL> separators = ( '<STR_LIT:U+002C>' , '<STR_LIT>' ) <EOL> ) <EOL> ) <EOL> except ValueError : <EOL> print "<STR_LIT>" , sstore <EOL> if tb is not None : <EOL> f . write ( "<STR_LIT>" ) <EOL> f . write ( str ( traceback . extract_tb ( tb ) ) . replace ( "<STR_LIT>" , "<STR_LIT>" ) . replace ( "<STR_LIT>" , "<STR_LIT>" ) . replace ( "<STR_LIT>" , "<STR_LIT>" ) ) <EOL> f . flush ( ) <EOL> f . close ( ) <EOL> f = open ( filename + "<STR_LIT>" , "<STR_LIT:wb>" ) <EOL> cPickle . dump ( self . store , f ) <EOL> f . close ( ) <EOL> signal . signal ( signal . SIGTERM , _handler_sig_term ) <EOL> if MSET . VERBOSE : <EOL> print "<STR_LIT:\n>" + "<STR_LIT>" <EOL> MCAN . friendly ( "<STR_LIT>" , "<STR_LIT>" % os . getpid ( ) ) <EOL> if recorder == "<STR_LIT:default>" : <EOL> params = { <EOL> "<STR_LIT>" : <NUM_LIT:1> , <EOL> "<STR_LIT>" : <NUM_LIT:1> <EOL> } <EOL> recorder = MREC . GGPlot2 ( runName , ** params ) <EOL> MCAN . friendly ( <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" % ( params , recorder . filename ) <EOL> ) <EOL> if not self . saveIfMurdered : <EOL> return self . run ( runName , model , recorder , * args , ** kwargs ) <EOL> else : <EOL> try : <EOL> return self . run ( runName , model , recorder , * args , ** kwargs ) <EOL> except MSTOP . EndOfTraining as e : <EOL> print e . message <EOL> death_time = time . ctime ( ) . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:_>' ) <EOL> filename = "<STR_LIT>" + runName + "<STR_LIT:_>" + death_time <EOL> f = open ( filename + "<STR_LIT>" , '<STR_LIT:w>' ) <EOL> f . write ( "<STR_LIT>" % runName ) <EOL> f . write ( "<STR_LIT>" % os . getpid ( ) ) <EOL> f . write ( "<STR_LIT>" % death_time ) <EOL> f . write ( "<STR_LIT>" % self . store [ "<STR_LIT>" ] [ "<STR_LIT>" ] ) <EOL> f . write ( "<STR_LIT>" % e . stopCriterion . name ) <EOL> f . write ( "<STR_LIT>" % e . message ) <EOL> sstore = str ( self . store ) . replace ( "<STR_LIT:'>" , '<STR_LIT:">' ) . replace ( "<STR_LIT:True>" , '<STR_LIT:true>' ) . replace ( "<STR_LIT:False>" , '<STR_LIT:false>' ) <EOL> f . write ( <EOL> "<STR_LIT>" % json . dumps ( <EOL> json . loads ( sstore ) , sort_keys = True , <EOL> indent = <NUM_LIT:4> , <EOL> separators = ( '<STR_LIT:U+002C>' , '<STR_LIT>' ) <EOL> ) <EOL> ) <EOL> f . flush ( ) <EOL> f . close ( ) <EOL> model . save ( filename ) <EOL> f = open ( filename + "<STR_LIT>" , "<STR_LIT:wb>" ) <EOL> cPickle . dump ( self . store , f ) <EOL> f . close ( ) <EOL> except KeyboardInterrupt : <EOL> exType , ex , tb = sys . exc_info ( ) <EOL> _dieGracefully ( exType , tb ) <EOL> raise <EOL> except : <EOL> exType , ex , tb = sys . exc_info ( ) <EOL> _dieGracefully ( exType , tb ) <EOL> raise <EOL> def run ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplemented ( "<STR_LIT>" ) <EOL> class DefaultTrainer ( Trainer_ABC ) : <EOL> """<STR_LIT>""" <EOL> SEQUENTIAL_TRAINING = <NUM_LIT:0> <EOL> SIMULTANEOUS_TRAINING = <NUM_LIT:1> <EOL> RANDOM_PICK_TRAINING = <NUM_LIT:2> <EOL> ALL_SET = - <NUM_LIT:1> <EOL> def __init__ ( self , <EOL> trainMaps , <EOL> testMaps , <EOL> validationMaps , <EOL> trainMiniBatchSize , <EOL> stopCriteria = [ ] , <EOL> testMiniBatchSize = - <NUM_LIT:1> , <EOL> validationMiniBatchSize = - <NUM_LIT:1> , <EOL> saveIfMurdered = True ) : <EOL> """<STR_LIT>""" <EOL> Trainer_ABC . __init__ ( self ) <EOL> self . maps = { <EOL> "<STR_LIT:train>" : trainMaps , <EOL> "<STR_LIT:test>" : testMaps , <EOL> } <EOL> if validationMaps is not None : <EOL> self . maps [ "<STR_LIT>" ] = validationMaps <EOL> self . miniBatchSizes = { <EOL> "<STR_LIT:train>" : trainMiniBatchSize , <EOL> "<STR_LIT:test>" : testMiniBatchSize , <EOL> "<STR_LIT>" : validationMiniBatchSize <EOL> } <EOL> self . stopCriteria = stopCriteria <EOL> self . saveIfMurdered = saveIfMurdered <EOL> self . trainingOrdersHR = { <EOL> self . SIMULTANEOUS_TRAINING : "<STR_LIT>" , <EOL> self . SEQUENTIAL_TRAINING : "<STR_LIT>" , <EOL> self . RANDOM_PICK_TRAINING : "<STR_LIT>" <EOL> } <EOL> def start ( self , runName , model , recorder = "<STR_LIT:default>" , trainingOrder = <NUM_LIT:0> , moreHyperParameters = { } ) : <EOL> """<STR_LIT>""" <EOL> Trainer_ABC . start ( self , runName , model , recorder , trainingOrder , moreHyperParameters ) <EOL> def run ( self , name , model , recorder , trainingOrder , moreHyperParameters ) : <EOL> """<STR_LIT>""" <EOL> def setHPs ( layer , thing , dct ) : <EOL> try : <EOL> thingObj = getattr ( l , thing ) <EOL> except AttributeError : <EOL> return <EOL> if thingObj is not None : <EOL> if type ( thingObj ) is types . ListType : <EOL> for obj in thingObj : <EOL> if len ( obj . hyperParameters ) == <NUM_LIT:0> : <EOL> dct [ "<STR_LIT>" % ( l . name , thing , obj . name ) ] = <NUM_LIT:1> <EOL> else : <EOL> for hp in obj . hyperParameters : <EOL> dct [ "<STR_LIT>" % ( l . name , thing , obj . name , hp ) ] = getattr ( obj , hp ) <EOL> else : <EOL> if len ( thingObj . hyperParameters ) == <NUM_LIT:0> : <EOL> dct [ "<STR_LIT>" % ( l . name , thingObj . name ) ] = <NUM_LIT:1> <EOL> else : <EOL> for hp in thingObj . hyperParameters : <EOL> dct [ "<STR_LIT>" % ( l . name , hp ) ] = getattr ( thingObj , hp ) <EOL> def _trainTest ( aMap , modelFct , trainingOrder , miniBatchSize , inputLayers , outputLayers ) : <EOL> scores = { } <EOL> layerList = inputLayers <EOL> if miniBatchSize == DefaultTrainer . ALL_SET : <EOL> for output in outputLayers : <EOL> layerList . append ( output ) <EOL> kwargs = dict ( aMap . getAll ( layerList = layerList ) ) <EOL> res = modelFct ( output , ** kwargs ) <EOL> scores [ output . name ] = res [ <NUM_LIT:0> ] <EOL> layerList . pop ( - <NUM_LIT:1> ) <EOL> else : <EOL> if trainingOrder == DefaultTrainer . SEQUENTIAL_TRAINING : <EOL> for output in outputLayers : <EOL> for i in xrange ( <NUM_LIT:0> , len ( aMap ) , miniBatchSize ) : <EOL> layerList . append ( output ) <EOL> batchData = aMap . getBatch ( i , miniBatchSize , layerList = layerList ) <EOL> res = modelFct ( output , ** batchData ) <EOL> try : <EOL> scores [ output . name ] . append ( res [ <NUM_LIT:0> ] ) <EOL> except KeyError : <EOL> scores [ output . name ] = [ res [ <NUM_LIT:0> ] ] <EOL> layerList . pop ( - <NUM_LIT:1> ) <EOL> elif trainingOrder == DefaultTrainer . SIMULTANEOUS_TRAINING : <EOL> for i in xrange ( <NUM_LIT:0> , len ( aMap ) , miniBatchSize ) : <EOL> batchData = aMap . getBatch ( i , miniBatchSize , layerList = layerList ) <EOL> for output in outputLayers : <EOL> layerList . append ( output ) <EOL> res = modelFct ( output , ** batchData ) <EOL> try : <EOL> scores [ output . name ] . append ( res [ <NUM_LIT:0> ] ) <EOL> except KeyError : <EOL> scores [ output . name ] = [ res [ <NUM_LIT:0> ] ] <EOL> layerList . pop ( - <NUM_LIT:1> ) <EOL> elif trainingOrder == DefaultTrainer . RANDOM_PICK_TRAINING : <EOL> outputOrder = list ( numpy . random . randint ( <NUM_LIT:0> , len ( outputLayers ) , len ( aMap ) / miniBatchSize + <NUM_LIT:1> ) ) <EOL> for i in xrange ( <NUM_LIT:0> , len ( aMap ) , miniBatchSize ) : <EOL> output = outputLayers [ outputOrder . pop ( ) ] <EOL> layerList . append ( output ) <EOL> batchData = aMap . getBatch ( i , miniBatchSize , layerList = layerList ) <EOL> res = modelFct ( output , ** batchData ) <EOL> try : <EOL> scores [ output . name ] . append ( res [ <NUM_LIT:0> ] ) <EOL> except KeyError : <EOL> scores [ output . name ] = [ res [ <NUM_LIT:0> ] ] <EOL> layerList . pop ( - <NUM_LIT:1> ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % trainingOrder ) <EOL> for outputName in scores : <EOL> scores [ outputName ] = numpy . mean ( scores [ outputName ] ) <EOL> if len ( scores ) > <NUM_LIT:1> : <EOL> scores [ "<STR_LIT>" ] = numpy . mean ( scores . values ( ) ) <EOL> return scores <EOL> if trainingOrder not in self . trainingOrdersHR : <EOL> raise ValueError ( "<STR_LIT>" % trainingOrder ) <EOL> legend = [ "<STR_LIT:name>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> legend . extend ( moreHyperParameters . keys ( ) ) <EOL> hyperParameters = OrderedDict ( ) <EOL> for l in model . layers . itervalues ( ) : <EOL> hyperParameters [ "<STR_LIT>" % l . name ] = len ( l ) <EOL> try : <EOL> hyperParameters [ "<STR_LIT>" ] = l . activation . __name__ <EOL> except AttributeError : <EOL> pass <EOL> setHPs ( l , "<STR_LIT>" , hyperParameters ) <EOL> setHPs ( l , "<STR_LIT>" , hyperParameters ) <EOL> setHPs ( l , "<STR_LIT>" , hyperParameters ) <EOL> if l . type == MNET . TYPE_OUTPUT_LAYER : <EOL> setHPs ( l , "<STR_LIT>" , hyperParameters ) <EOL> legend . extend ( hyperParameters . keys ( ) ) <EOL> self . store [ "<STR_LIT>" ] . update ( hyperParameters ) <EOL> self . store [ "<STR_LIT>" ] . update ( moreHyperParameters ) <EOL> self . store [ "<STR_LIT>" ] [ "<STR_LIT:name>" ] = name <EOL> self . store [ "<STR_LIT>" ] [ "<STR_LIT>" ] = self . trainingOrdersHR [ trainingOrder ] <EOL> self . store [ "<STR_LIT>" ] [ "<STR_LIT>" ] = os . getpid ( ) <EOL> self . recorder = recorder <EOL> startTime = time . time ( ) <EOL> self . store [ "<STR_LIT>" ] [ "<STR_LIT>" ] = <NUM_LIT:0> <EOL> inputLayers = model . inputs . values ( ) <EOL> outputLayers = model . outputs . values ( ) <EOL> for mapName in self . maps : <EOL> self . store [ "<STR_LIT>" ] [ mapName ] = self . maps [ mapName ] . getMinFullLength ( ) <EOL> self . store [ "<STR_LIT>" ] [ mapName ] = { } <EOL> for o in outputLayers : <EOL> self . store [ "<STR_LIT>" ] [ mapName ] [ o . name ] = "<STR_LIT>" <EOL> while True : <EOL> for mapName in [ "<STR_LIT:train>" , "<STR_LIT:test>" , "<STR_LIT>" ] : <EOL> try : <EOL> aMap = self . maps [ mapName ] <EOL> if len ( aMap ) > <NUM_LIT:0> : <EOL> scores = { } <EOL> aMap . commit ( ) <EOL> if mapName == "<STR_LIT:train>" : <EOL> modelFct = model . train <EOL> else : <EOL> modelFct = model . test <EOL> scores = _trainTest ( <EOL> aMap , <EOL> modelFct , <EOL> trainingOrder , <EOL> self . miniBatchSizes [ mapName ] , <EOL> inputLayers , <EOL> outputLayers <EOL> ) <EOL> for k , v in scores . iteritems ( ) : <EOL> self . store [ "<STR_LIT>" ] [ mapName ] [ k ] = v <EOL> except KeyError : <EOL> pass <EOL> runtime = ( time . time ( ) - startTime ) / <NUM_LIT> <EOL> self . store [ "<STR_LIT>" ] . update ( ( <EOL> ( "<STR_LIT>" , runtime ) , <EOL> ) ) <EOL> self . recorder . commit ( self . store , model ) <EOL> for crit in self . stopCriteria : <EOL> if crit . stop ( self ) : <EOL> raise MSTOP . EndOfTraining ( crit ) <EOL> for l in model . layers . itervalues ( ) : <EOL> try : <EOL> l . learningScenario . update ( self . store ) <EOL> except AttributeError : <EOL> pass <EOL> self . store [ "<STR_LIT>" ] [ "<STR_LIT>" ] += <NUM_LIT:1> <EOL> def runCustom ( self , model , outputName , functionName , dataMapName , miniBatchSize = - <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> modelFct = getattr ( model , functionName ) <EOL> aMap = self . maps [ dataMapName ] <EOL> layerList = [ outputName ] <EOL> for l in model . inputs . itervalues ( ) : <EOL> layerList . append ( l . name ) <EOL> if miniBatchSize == DefaultTrainer . ALL_SET : <EOL> kwargs = dict ( aMap . getAll ( layerList = layerList ) ) <EOL> res = modelFct ( outputName , ** kwargs ) <EOL> else : <EOL> for i in xrange ( <NUM_LIT:0> , len ( aMap ) , miniBatchSize ) : <EOL> batchData = aMap . getBatch ( i , miniBatchSize , layerList = layerList ) <EOL> res = modelFct ( outputName , ** batchData ) <EOL> return res [ <NUM_LIT:0> ] </s>
