<s> """<STR_LIT>""" <EOL> from abc import ABCMeta , abstractmethod <EOL> import numpy as np <EOL> import logging <EOL> __copyright__ = "<STR_LIT>" <EOL> __credits__ = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ] <EOL> __license__ = "<STR_LIT>" <EOL> __author__ = "<STR_LIT>" <EOL> class Agent ( object ) : <EOL> """<STR_LIT>""" <EOL> __metaclass__ = ABCMeta <EOL> representation = None <EOL> discount_factor = None <EOL> policy = None <EOL> eligibility_trace = [ ] <EOL> logger = None <EOL> episode_count = <NUM_LIT:0> <EOL> random_state = None <EOL> def __init__ ( self , policy , representation , discount_factor , seed = <NUM_LIT:1> , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . representation = representation <EOL> self . policy = policy <EOL> self . discount_factor = discount_factor <EOL> self . logger = logging . getLogger ( "<STR_LIT>" + self . __class__ . __name__ ) <EOL> self . random_state = np . random . RandomState ( seed = seed ) <EOL> def init_randomization ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> @ abstractmethod <EOL> def learn ( self , s , p_actions , a , r , ns , np_actions , na , terminal ) : <EOL> """<STR_LIT>""" <EOL> return NotImplementedError <EOL> def episodeTerminated ( self ) : <EOL> """<STR_LIT>""" <EOL> self . episode_count += <NUM_LIT:1> <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> self . eligibility_trace = np . zeros_like ( self . eligibility_trace ) <EOL> class DescentAlgorithm ( object ) : <EOL> """<STR_LIT>""" <EOL> initial_learn_rate = <NUM_LIT:0.1> <EOL> learn_rate = <NUM_LIT:0> <EOL> eligibility_trace = [ ] <EOL> logger = None <EOL> episode_count = <NUM_LIT:0> <EOL> learn_rate_decay_mode = '<STR_LIT>' <EOL> valid_decay_modes = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> boyan_N0 = <NUM_LIT:1000> <EOL> def __init__ ( self , initial_learn_rate = <NUM_LIT:0.1> , learn_rate_decay_mode = '<STR_LIT>' , boyan_N0 = <NUM_LIT:1000> , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . initial_learn_rate = initial_learn_rate <EOL> self . learn_rate = initial_learn_rate <EOL> self . learn_rate_decay_mode = learn_rate_decay_mode . lower ( ) <EOL> self . boyan_N0 = boyan_N0 <EOL> if self . learn_rate_decay_mode == '<STR_LIT>' : <EOL> self . initial_learn_rate = <NUM_LIT:1.0> <EOL> self . learn_rate = <NUM_LIT:1.0> <EOL> super ( DescentAlgorithm , self ) . __init__ ( ** kwargs ) <EOL> def updateLearnRate ( self , phi , phi_prime , eligibility_trace , <EOL> discount_factor , nnz , terminal ) : <EOL> """<STR_LIT>""" <EOL> if self . learn_rate_decay_mode == '<STR_LIT>' : <EOL> if not terminal : <EOL> candid_learn_rate = np . dot ( discount_factor * phi_prime - phi , <EOL> eligibility_trace ) <EOL> if candid_learn_rate < <NUM_LIT:0> : <EOL> self . learn_rate = np . minimum ( self . learn_rate , - <NUM_LIT:1.0> / candid_learn_rate ) <EOL> elif self . learn_rate_decay_mode == '<STR_LIT>' : <EOL> self . learn_rate = self . initial_learn_rate * ( self . boyan_N0 + <NUM_LIT:1.> ) / ( self . boyan_N0 + ( self . episode_count + <NUM_LIT:1> ) ** <NUM_LIT> ) <EOL> self . learn_rate /= np . sum ( np . abs ( phi ) ) <EOL> elif self . learn_rate_decay_mode == '<STR_LIT>' : <EOL> self . learn_rate = self . initial_learn_rate * ( self . boyan_N0 + <NUM_LIT:1.> ) / ( self . boyan_N0 + ( self . episode_count + <NUM_LIT:1> ) ** <NUM_LIT> ) <EOL> elif self . learn_rate_decay_mode == "<STR_LIT>" : <EOL> self . learn_rate = self . initial_learn_rate <EOL> else : <EOL> self . logger . warn ( "<STR_LIT>" ) <EOL> def episodeTerminated ( self ) : <EOL> """<STR_LIT>""" <EOL> self . episode_count += <NUM_LIT:1> <EOL> self . representation . episodeTerminated ( ) <EOL> super ( DescentAlgorithm , self ) . episodeTerminated ( ) </s>
