<s> import chainer . functions as F <EOL> import chainer . links as L <EOL> from chainer import Variable <EOL> from chainer . links import caffe <EOL> from chainer import computational_graph as c <EOL> from deel . tensor import * <EOL> from deel . network import * <EOL> import copy <EOL> from deel import * <EOL> import chainer <EOL> import json <EOL> import os <EOL> import multiprocessing <EOL> import threading <EOL> import time <EOL> import six <EOL> import numpy as np <EOL> import os . path <EOL> from PIL import Image <EOL> from six . moves import queue <EOL> import pickle <EOL> import hashlib <EOL> import datetime <EOL> import sys <EOL> import random <EOL> from deel . agentServer import AgentServer <EOL> import deel . model . q_net <EOL> class DQN ( Network ) : <EOL> policyFrozen = False <EOL> epsilonDelta = <NUM_LIT:1.0> / <NUM_LIT:10> ** <NUM_LIT:4> <EOL> min_eps = <NUM_LIT:0.1> <EOL> actions = None <EOL> reward = None <EOL> def __init__ ( self , given_dim = ( <NUM_LIT> , <NUM_LIT:6> , <NUM_LIT:6> ) , actions = [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> ] , depth_image_dim = <NUM_LIT:0> ) : <EOL> super ( DQN , self ) . __init__ ( '<STR_LIT>' ) <EOL> self . actions = actions <EOL> self . age = <NUM_LIT:0> <EOL> self . depth_image_dim = depth_image_dim <EOL> self . image_feature_dim = getDim ( given_dim ) + depth_image_dim <EOL> print "<STR_LIT>" , self . image_feature_dim <EOL> self . time = <NUM_LIT:0> <EOL> self . epsilon = <NUM_LIT:1.0> <EOL> self . func = model . q_net . QNet ( Deel . gpu , self . actions , self . image_feature_dim ) <EOL> def actionAndLearn ( self , x = None ) : <EOL> if x is None : <EOL> x = Tensor . context <EOL> if AgentServer . mode == '<STR_LIT:start>' : <EOL> return self . start ( x ) <EOL> elif AgentServer . mode == '<STR_LIT>' : <EOL> return self . step ( x ) <EOL> elif AgentServer . mode == '<STR_LIT:end>' : <EOL> return self . end ( x ) <EOL> def start ( self , x = None ) : <EOL> if x is None : <EOL> x = Tensor . context <EOL> obs_array = x . content . data <EOL> self . state = np . zeros ( ( self . func . hist_size , self . image_feature_dim ) , dtype = np . uint8 ) <EOL> self . state [ <NUM_LIT:0> ] = obs_array <EOL> state_ = np . asanyarray ( self . state . reshape ( <NUM_LIT:1> , self . func . hist_size , self . image_feature_dim ) , dtype = np . float32 ) <EOL> if Deel . gpu >= <NUM_LIT:0> : <EOL> state_ = cuda . to_gpu ( state_ ) <EOL> action , Q_now = self . func . e_greedy ( state_ , self . epsilon ) <EOL> returnAction = action <EOL> self . lastAction = copy . deepcopy ( returnAction ) <EOL> self . last_state = self . state . copy ( ) <EOL> self . last_observation = obs_array <EOL> return returnAction <EOL> def step ( self , x = None ) : <EOL> if x is None : <EOL> x = Tensor . context <EOL> obs_array = x . content . data <EOL> obs_processed = np . maximum ( obs_array , self . last_observation ) <EOL> if self . func . hist_size == <NUM_LIT:4> : <EOL> self . state = np . asanyarray ( [ self . state [ <NUM_LIT:1> ] , self . state [ <NUM_LIT:2> ] , self . state [ <NUM_LIT:3> ] , obs_processed ] , dtype = np . uint8 ) <EOL> elif self . func . hist_size == <NUM_LIT:2> : <EOL> self . state = np . asanyarray ( [ self . state [ <NUM_LIT:1> ] , obs_processed ] , dtype = np . uint8 ) <EOL> elif self . func . hist_size == <NUM_LIT:1> : <EOL> self . state = np . asanyarray ( [ obs_processed ] , dtype = np . uint8 ) <EOL> else : <EOL> print ( "<STR_LIT>" ) <EOL> state_ = np . asanyarray ( self . state . reshape ( <NUM_LIT:1> , self . func . hist_size , self . image_feature_dim ) , dtype = np . float32 ) <EOL> if Deel . gpu >= <NUM_LIT:0> : <EOL> state_ = cuda . to_gpu ( state_ ) <EOL> if self . policyFrozen is False : <EOL> if self . func . initial_exploration < self . time : <EOL> self . epsilon -= self . epsilonDelta <EOL> if self . epsilon < self . min_eps : <EOL> self . epsilon = self . min_eps <EOL> eps = self . epsilon <EOL> else : <EOL> print "<STR_LIT>" % ( self . time , self . func . initial_exploration ) <EOL> eps = <NUM_LIT:1.0> <EOL> else : <EOL> print "<STR_LIT>" <EOL> eps = <NUM_LIT> <EOL> action , Q_now = self . func . e_greedy ( state_ , eps ) <EOL> return self , action , eps , Q_now , obs_array <EOL> def step_after ( self , reward , action , eps , q_now , obs_array ) : <EOL> '''<STR_LIT>''' <EOL> self . reward = reward <EOL> if self . policyFrozen is False : <EOL> self . func . stock_experience ( self . time , self . last_state , self . lastAction , reward , self . state , False ) <EOL> self . func . experience_replay ( self . time ) <EOL> if self . func . initial_exploration < self . time and np . mod ( self . time , self . func . target_model_update_freq ) == <NUM_LIT:0> : <EOL> print '<STR_LIT>' <EOL> self . func . target_model_update ( ) <EOL> if Deel . gpu >= <NUM_LIT:0> : <EOL> q_max = np . max ( q_now . get ( ) ) <EOL> else : <EOL> q_max = np . max ( q_now ) <EOL> print '<STR_LIT>' % ( <EOL> self . time , self . func . action_to_index ( action ) , reward , eps , q_max ) <EOL> self . last_observation = obs_array <EOL> if self . policyFrozen is False : <EOL> self . lastAction = copy . deepcopy ( action ) <EOL> self . last_state = self . state . copy ( ) <EOL> self . time += <NUM_LIT:1> <EOL> def end ( self , x ) : <EOL> reward = self . reward <EOL> print '<STR_LIT>' % ( reward , self . epsilon ) <EOL> if self . policyFrozen is False : <EOL> self . func . stock_experience ( self . time , self . last_state , self . lastAction , reward , self . last_state , <EOL> True ) <EOL> self . func . experience_replay ( self . time ) <EOL> if self . func . initial_exploration < self . time and np . mod ( self . time , self . func . target_model_update_freq ) == <NUM_LIT:0> : <EOL> print '<STR_LIT>' <EOL> self . func . target_model_update ( ) <EOL> if self . policyFrozen is False : <EOL> self . time += <NUM_LIT:1> <EOL> self . age += <NUM_LIT:1> <EOL> if self . age % <NUM_LIT:10> == <NUM_LIT:0> : <EOL> self . save ( '<STR_LIT>' % self . age ) <EOL> def save ( self , filename ) : <EOL> pickle . dump ( self . func . model , open ( filename , '<STR_LIT:wb>' ) ) <EOL> def load ( self , filename ) : <EOL> self . func . model = pickle . load ( open ( filename , '<STR_LIT:rb>' ) ) </s>
