<s> '''<STR_LIT>''' <EOL> from numpy import * <EOL> import scipy as Sci <EOL> import scipy . linalg <EOL> import copy <EOL> import matplotlib . pyplot as pyplot <EOL> class simple_RL ( object ) : <EOL> def __init__ ( self , num_states , alpha , gamma , goal_state ) : <EOL> self . num_states = num_states <EOL> self . alpha = alpha <EOL> self . gamma = gamma <EOL> self . num_actions = <NUM_LIT:4> <EOL> self . Q_mat = zeros ( ( self . num_states [ <NUM_LIT:0> ] * self . num_states [ <NUM_LIT:1> ] , self . num_actions ) ) <EOL> self . goal_state = goal_state <EOL> self . trans_dict = { <NUM_LIT:0> : ( <NUM_LIT:0> , - <NUM_LIT:1> ) , <NUM_LIT:1> : ( <NUM_LIT:1> , <NUM_LIT:1> ) , <NUM_LIT:2> : ( <NUM_LIT:0> , <NUM_LIT:1> ) , <NUM_LIT:3> : ( <NUM_LIT:1> , - <NUM_LIT:1> ) } <EOL> self . start_state = [ random . randint ( <NUM_LIT:0> , num_states [ <NUM_LIT:0> ] ) , random . randint ( <NUM_LIT:0> , num_states [ <NUM_LIT:1> ] ) ] <EOL> self . curr_state = self . start_state <EOL> def execute_action ( self , action ) : <EOL> move = self . trans_dict [ action ] <EOL> prev_state = copy . deepcopy ( self . curr_state ) <EOL> self . curr_state [ move [ <NUM_LIT:0> ] ] = min ( max ( self . curr_state [ move [ <NUM_LIT:0> ] ] + move [ <NUM_LIT:1> ] , <NUM_LIT:0> ) , self . num_states [ move [ <NUM_LIT:0> ] ] - <NUM_LIT:1> ) <EOL> r = self . get_reward ( self . curr_state ) <EOL> self . Q_mat [ self . num_states [ <NUM_LIT:0> ] * ( prev_state [ <NUM_LIT:0> ] ) + prev_state [ <NUM_LIT:1> ] , action ] = ( <EOL> self . Q_mat [ self . num_states [ <NUM_LIT:0> ] * ( prev_state [ <NUM_LIT:0> ] ) + prev_state [ <NUM_LIT:1> ] , action ] * ( <NUM_LIT:1> - self . alpha ) + <EOL> ( r + self . gamma * max ( self . Q_mat [ self . num_states [ <NUM_LIT:0> ] * ( self . curr_state [ <NUM_LIT:0> ] ) + ( self . curr_state [ <NUM_LIT:1> ] ) ] ) ) * self . alpha ) <EOL> if tuple ( self . curr_state ) == self . goal_state : <EOL> return True <EOL> return False <EOL> def get_reward ( self , state ) : <EOL> if tuple ( state ) == self . goal_state : <EOL> return <NUM_LIT:10> <EOL> return - <NUM_LIT:1> <EOL> def run ( self , num_trials ) : <EOL> tot_reward = [ ] <EOL> tot_steps = [ ] <EOL> tri_reward = <NUM_LIT:0> <EOL> num_steps = <NUM_LIT:0> <EOL> for x in xrange ( num_trials ) : <EOL> if x % <NUM_LIT:10> == <NUM_LIT:0> and x >= <NUM_LIT:10> : <EOL> print '<STR_LIT>' , x , ( sum ( tot_reward [ x - <NUM_LIT:10> : x ] ) / <NUM_LIT:10> ) , ( sum ( tot_steps [ x - <NUM_LIT:10> : x ] ) / <NUM_LIT:10> ) <EOL> self . curr_state = [ random . randint ( <NUM_LIT:0> , self . num_states [ <NUM_LIT:0> ] ) , random . randint ( <NUM_LIT:0> , self . num_states [ <NUM_LIT:1> ] ) ] <EOL> tri_reward = <NUM_LIT:0> <EOL> num_steps = <NUM_LIT:0> <EOL> not_done = True <EOL> path = [ ] <EOL> while ( not_done ) : <EOL> num_steps += <NUM_LIT:1> <EOL> best_action = nonzero ( self . Q_mat [ self . num_states [ <NUM_LIT:0> ] * ( self . curr_state [ <NUM_LIT:0> ] ) + ( self . curr_state [ <NUM_LIT:1> ] ) ] == max ( self . Q_mat [ self . num_states [ <NUM_LIT:0> ] * ( self . curr_state [ <NUM_LIT:0> ] ) + ( self . curr_state [ <NUM_LIT:1> ] ) ] ) ) [ <NUM_LIT:0> ] <EOL> next_action = best_action [ random . randint ( <NUM_LIT:0> , len ( best_action ) ) ] if random . random ( ) < <NUM_LIT> else random . randint ( <NUM_LIT:0> , self . num_actions ) <EOL> not_done = not self . execute_action ( next_action ) <EOL> tri_reward += self . get_reward ( self . curr_state ) <EOL> path . append ( copy . deepcopy ( self . curr_state ) ) <EOL> tot_reward . append ( tri_reward ) <EOL> tot_steps . append ( num_steps ) <EOL> fig1 = pyplot . figure ( <NUM_LIT:1> ) <EOL> pyplot . plot ( [ ( sum ( tot_reward [ x - <NUM_LIT:100> : x ] ) / <NUM_LIT:100> ) if x >= <NUM_LIT:100> else ( sum ( tot_reward [ <NUM_LIT:0> : x ] ) / x ) for x in xrange ( num_trials ) ] ) <EOL> fig1 . suptitle ( "<STR_LIT>" ) <EOL> fig2 = pyplot . figure ( <NUM_LIT:2> ) <EOL> pyplot . plot ( [ ( sum ( tot_steps [ x - <NUM_LIT:100> : x ] ) / <NUM_LIT:100> ) if x >= <NUM_LIT:100> else ( sum ( tot_steps [ <NUM_LIT:0> : x ] ) / x ) for x in xrange ( num_trials ) ] ) <EOL> fig2 . suptitle ( "<STR_LIT>" ) <EOL> pyplot . show ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> rl = simple_RL ( ( <NUM_LIT> , <NUM_LIT> ) , <NUM_LIT> , <NUM_LIT> , ( <NUM_LIT:7> , <NUM_LIT:6> ) ) <EOL> rl . run ( <NUM_LIT:1000> ) <EOL> print rl . Q_mat </s>
