<s> from nn import NeuralNet <EOL> import random <EOL> import numpy as np <EOL> class Experience ( ) : <EOL> def __init__ ( self , s1 , a1 , r , s2 , a2 ) : <EOL> '''<STR_LIT>''' <EOL> self . s1 = s1 <EOL> self . s2 = s2 <EOL> self . a1 = a1 <EOL> self . a2 = a2 <EOL> self . r = r <EOL> class QNN ( ) : <EOL> """<STR_LIT>""" <EOL> def __call__ ( self , s , a = None ) : <EOL> """<STR_LIT>""" <EOL> if a == None : <EOL> return self . GetValue ( s ) <EOL> return self . GetValue ( s , a ) <EOL> def __init__ ( self , nactions , input_size , max_experiences = <NUM_LIT> , gamma = <NUM_LIT> , alpha = <NUM_LIT:0.1> , use_sarsa = False ) : <EOL> lay = [ input_size , int ( ( nactions + input_size ) / <NUM_LIT> ) , int ( ( nactions + input_size ) / <NUM_LIT> ) , nactions ] <EOL> self . nactions = nactions <EOL> self . NN = NeuralNet ( layers = lay , epsilon = <NUM_LIT> , learningRate = alpha ) <EOL> self . experiences = [ ] <EOL> self . max_experiences = max_experiences <EOL> self . gamma = gamma <EOL> self . use_sarsa = use_sarsa <EOL> self . prob_remember = <NUM_LIT:0.1> <EOL> self . num_replay_samples = <NUM_LIT:10> <EOL> def GetValue ( self , s , a = None ) : <EOL> """<STR_LIT>""" <EOL> out = self . NN . propagate ( s ) <EOL> if ( a == None ) : <EOL> return out <EOL> return out [ a ] <EOL> def Update ( self , s1 , a1 , r , s2 , a2 ) : <EOL> """<STR_LIT>""" <EOL> if ( self . use_sarsa ) : <EOL> v = r + self . gamma * self . GetValue ( s2 , a2 ) <EOL> else : <EOL> v = r + self . gamma * max ( self . GetValue ( s2 ) ) <EOL> a = np . zeros ( self . nactions ) <EOL> a [ a1 ] = v <EOL> self . NN . propagateAndUpdate ( s1 , a ) <EOL> def RememberExperience ( self , s1 , a1 , r , s2 , a2 ) : <EOL> if ( random . random ( ) < self . prob_remember ) : <EOL> if ( len ( self . experiences ) >= self . max_experiences ) : <EOL> self . experiences . pop ( random . randint ( <NUM_LIT:0> , self . max_experiences - <NUM_LIT:1> ) ) <EOL> self . experiences . append ( Experience ( s1 , a1 , r , s2 , a2 ) ) <EOL> def ExperienceReplay ( self ) : <EOL> if ( len ( self . experiences ) < self . num_replay_samples ) : <EOL> return <EOL> for i in xrange ( self . num_replay_samples ) : <EOL> index = random . randint ( <NUM_LIT:0> , len ( self . experiences ) - <NUM_LIT:1> ) <EOL> exp = self . experiences [ index ] <EOL> self . Update ( exp . s1 , exp . a1 , exp . r , exp . s2 , exp . a2 ) </s>
