###### Formulation:
* Stages: $i = 1,2,3,\cdots$.
* State Space: $\mathcal{S} = \{0,1,2,3,4,5,6,7\}$.

| x | x   | x  | x   | x
| ---  | --- | --- | --- | ---
| 0| x   | 4   | x | 7   
| 1  | 2 | 3 |5 | 6 


* Control Space: $\mathcal{A} = \{u, d, l, r\}$
* Feasible control mapping $\mathcal{S}\rightarrow\mathcal{A}: \mathcal{A}(x)=\{u, d, l, r\}$
* Transition mapping, given $s_i, a_i$, $\mathcal{S}\rightarrow [0,1]$, for $i\in \{1,2,\cdots\}$. 



* Reward function $\mathcal{S}\times\mathcal{A}\rightarrow\mathbb{R}$: 
Executing "up" at 4 gains a reward of 10, walking in side maze takes -1 each step. Hitting the wall takes -3.

Escape:
$$r(4, u)=10$$
Walking in maze:

* Let $v^*(x)$ be the optimal value function for state $x$, so it satisfies the following equation:

\begin{align}
    v^*(x_i) = &\max_{u\in \mathcal{U}(x)}\{r(x_i, u)+\alpha\mathbb{E}[v^*(x_{i+1})]\}\\
\end{align}

And it can be solved by value iteration:

\begin{align}
    v^{k+1}(x_i) = \mathcal{D}(v)(x_i) = &\max_{u\in \mathcal{U}(x)}\{r(x_i, u)+\alpha\mathbb{E}[v^k(x_{i+1})]\}\\
\end{align}
with arbitrary $v^0$, since $\mathcal{D}$ is a contraction mapping.


```python
from itertools import product
import numpy as np
import pandas as pd
from copy import deepcopy
import random
import matplotlib.pyplot as plt
import seaborn as sns
```


```python

```


```python
state_space = [0,1,2,3,4,5,6,7]
control_space = [0,1,2,3] # u,d,l,r
graph = {
    0: [-1, 1, -1, -1],
    1: [0 , -1, -1, 2],
    2: [-1, -1, 1, 3],
    3: [4, -1, 2, 5 ],
    4: [11, 3, -1, -1],
    5: [-1, -1, 3, 6],
    6: [7, -1, 5, -1],
    7: [-1, 6, -1, -1]
}
alpha = 0.95

def trans_prob(next_x, x, u):
    if (x, u) == (4, 0):
        return 0.125
    elif graph[x][u] >= 0:
        if graph[x][u] == next_x:
            return 1.
        else:
            return 0.
    else: 
        if next_x == x:
            return 1.
        else:
            return 0.


def reward(x, u):
    if (x, u) == (4, 0):
        return 10
    elif graph[x][u] < 0:
        return -3
    else:
        return -1
    
def display_pi(pi):
    di = {0:"u", 1: "d", 2: "l", 3:"r"}
    a = lambda x: di[pi[x]]
    ret = f'''
    -----   -----   -----
    | {a(0)} |   | {a(4)} |   | {a(7)} |
    ---------------------
    | {a(1)} | {a(2)} | {a(3)} | {a(5)} | {a(6)} |
    ---------------------
    '''
    print(ret)
```


```python
v = pd.Series([0.0 for _ in state_space], index = state_space)
pi = pd.Series([0 for _ in state_space], index = state_space)
```


```python
def value_iteration(state_space, control_space, trans_prob, reward, alpha, v, pi, MAX_ITER = 1000000, threshold = 1e-6):
    for _ in range(MAX_ITER):
        next_v = deepcopy(v)
        for x in state_space:
            tmp = [reward(x, u)+alpha*
                   np.sum([v[next_x]*trans_prob(next_x, x, u) for next_x in state_space]) 
                   for u in control_space]
            next_v[x]= np.max(tmp)
            pi[x]= control_space[np.argmax(tmp)]
        if abs(next_v-v).values.max()<1e-6:
            break
        else:
            v = next_v
            
    return v, pi
```


```python
v, pi = value_iteration(state_space, control_space, trans_prob, reward, 0.95, v, pi)
```


```python
v
```




    0    37.059319
    1    40.062442
    2    43.223624
    3    46.551185
    4    50.053879
    5    43.223624
    6    40.062442
    7    37.059319
    dtype: float64




```python
pi
```




    0    1
    1    3
    2    3
    3    0
    4    0
    5    2
    6    2
    7    1
    dtype: int64




```python
display_pi(pi)
```

    
        -----   -----   -----
        | d |   | u |   | d |
        ---------------------
        | r | r | u | l | l |
        ---------------------
        



```python
list(product(state_space, control_space))
```




    [(0, 0),
     (0, 1),
     (0, 2),
     (0, 3),
     (1, 0),
     (1, 1),
     (1, 2),
     (1, 3),
     (2, 0),
     (2, 1),
     (2, 2),
     (2, 3),
     (3, 0),
     (3, 1),
     (3, 2),
     (3, 3),
     (4, 0),
     (4, 1),
     (4, 2),
     (4, 3),
     (5, 0),
     (5, 1),
     (5, 2),
     (5, 3),
     (6, 0),
     (6, 1),
     (6, 2),
     (6, 3),
     (7, 0),
     (7, 1),
     (7, 2),
     (7, 3)]




```python
# Augmented MDP
state_space_d = list(product(state_space, control_space))
control_space_d = control_space.copy()
alpha = 0.95

def trans_prob_d(next_i, i, u):
    if next_i[1]==u:
        return trans_prob(next_i[0], i[0], i[1])
    else:
        return 0.0

def reward_d(x, u):
    return reward(x[0], x[1])
```


```python
v_d = pd.Series([0.0 for _ in state_space_d], index = state_space_d)
pi_d = pd.Series([1 for _ in state_space_d], index = state_space_d)
```


```python
v_AMDP, pi_AMDP = value_iteration(state_space_d, control_space_d, trans_prob_d, reward_d, 0.95, v_d, pi_d)
```


```python
v_AMDP
```




    (0, 0)    17.421565
    (0, 1)    21.496385
    (0, 2)    17.421565
    (0, 3)    17.421565
    (1, 0)    19.421565
    (1, 1)    19.496385
    (1, 2)    19.496385
    (1, 3)    23.680406
    (2, 0)    21.680406
    (2, 1)    21.680406
    (2, 2)    21.496385
    (2, 3)    25.979376
    (3, 0)    28.399344
    (3, 1)    23.979376
    (3, 2)    23.680406
    (3, 3)    23.680406
    (4, 0)    30.946679
    (4, 1)    25.979376
    (4, 2)    26.399344
    (4, 3)    26.399344
    (5, 0)    21.680406
    (5, 1)    21.680406
    (5, 2)    25.979376
    (5, 3)    21.496385
    (6, 0)    19.421565
    (6, 1)    19.496385
    (6, 2)    23.680406
    (6, 3)    19.496385
    (7, 0)    17.421565
    (7, 1)    21.496385
    (7, 2)    17.421565
    (7, 3)    17.421565
    dtype: float64




```python
pi_AMDP
```




    (0, 0)    1
    (0, 1)    3
    (0, 2)    1
    (0, 3)    1
    (1, 0)    1
    (1, 1)    3
    (1, 2)    3
    (1, 3)    3
    (2, 0)    3
    (2, 1)    3
    (2, 2)    3
    (2, 3)    0
    (3, 0)    0
    (3, 1)    0
    (3, 2)    3
    (3, 3)    2
    (4, 0)    0
    (4, 1)    0
    (4, 2)    0
    (4, 3)    0
    (5, 0)    2
    (5, 1)    2
    (5, 2)    0
    (5, 3)    2
    (6, 0)    1
    (6, 1)    2
    (6, 2)    2
    (6, 3)    2
    (7, 0)    1
    (7, 1)    2
    (7, 2)    1
    (7, 3)    1
    dtype: int64




```python
pi_AMDP_ = lambda x: pi_AMDP[x]
```


```python
# Q_learning env
class Env:
    def __init__(self):
#         self.gamma = gamma
        self.obs = 0
        self.control_space = tuple(control_space)
        self.state_space = tuple(state_space)
        self.state_dim = 1
        self.action_dim = 1
        self.graph = graph
        
    def step(self, action, current_obs = None):
        
        obs = self.obs if current_obs is None else current_obs
        self.obs = self._next_state(obs, action)
        r = reward(obs, action)
#         r = reward((q,p), action)
        return self.obs, r, False, {}
    
    def reset(self, random=True):
        self.obs = 0 if not random else np.random.choice(self.state_space)
        return self.obs
    
    def _next_state(self, obs, action):
        next_s = self.graph[obs][action]
        next_s = obs if next_s < 0 else next_s
        next_s = self.reset() if next_s == 11 else next_s
        return next_s
      
```


```python
class Q_table:
    def __init__(self, q_data=None, random = False):
        self.control_space = control_space
        self.state_space = state_space
        if q_data is None:
            if not random:
                self.data = pd.DataFrame({x:[0.0]*len(self.control_space) for x in self.state_space},
                                         index = self.control_space) 
            else:
                self.data = pd.DataFrame({x:np.random.randn(len(self.control_space)) for x in self.state_space}, 
                                         index = self.control_space) 
        else:
#             assert isinstance(q, Q_table)
            self.data = q_data
        
    def __getitem__(self, key):
        x, u = key
        return self.data.loc[u,x]
    
    def __setitem__(self, key, item):
        x, u = key
        self.data.loc[u,x] = item
    
    def max_value_act(self, x):
        tmp = [self.data.loc[k,x] for k in self.control_space]
        return np.max(tmp), self.control_space[np.argmax(tmp)]
    
    def difference(self, q):
        assert isinstance(q, Q_table)
        return Q_table(self.data-q.data)
        
    def L_infty(self):
        return np.max(np.abs(self.data).values)
    
    def L_1(self):
        return np.mean(np.abs(self.data).values)
    
    def yield_v_pi(self):
        v = pd.Series([0.0 for _ in self.state_space], index = self.state_space)
        pi = pd.Series([0 for _ in self.state_space], index = self.state_space)
        for i in self.state_space:
            v[i], pi[i] = self.max_value_act(i)     
        return v, pi
        
    def __repr__(self):

        return self.data.__repr__()
    
class Pool:
    def __init__(self, length):
        self.ls = [None]*length
        self.len = length
        self.idx = 0
        self.isFull = False
        
    def append(self, item):
        self.ls[self.idx] = item
        self.idx = (self.idx + 1) % self.len
        if not self.isFull:
            if self.idx + 1 == self.len:
                self.isFull = True
                
    def shuffle(self):
        cp = deepcopy(self.ls)
        random.shuffle(cp)
        return cp
        
    def __getitem__(self, key):
        idx = (self.idx+key) % self.len
        return self.ls[idx]
    
    def __iter__(self):
        return iter(self.ls[self.idx:]+self.ls[:self.idx])
    
    def __repr__(self):
        return str(self.ls[self.idx:]+self.ls[:self.idx])
```


```python
# Q-learning with pool


class Q_learning:
    def __init__(self, q, env, epsilon = 0.1, MAX_ITER = 10000, episode_length = 100, threshold = 1e-3):
        self.q = q
        self.env = env
        self.epsilon = epsilon
        self.MAX_ITER = MAX_ITER
        self.episode_length = episode_length
        self.threshold = threshold
        self.pool = Pool(40)
        self.qs = []
        
    def train(self, lr = lambda *args: 0.1):
        q = self.q
        env = self.env
        epsilon = self.epsilon
        MAX_ITER = self.MAX_ITER
        episode_length = self.episode_length
        threshold = self.threshold
        for i in range(MAX_ITER):
            print(i)
            obs = env.reset(random=True)
            r_ls = []
            for _ in range(10):
                action, is_random = (np.random.choice(control_space), True) \
                                    if np.random.uniform() < epsilon else \
                                    (q.max_value_act(obs)[1], False)
                next_obs, reward, _, _ = env.step(action)
                self.pool.append((obs, next_obs, action, reward, is_random))
                r_ls.append(reward)
                obs = next_obs
            if not self.pool.isFull:
                continue
            print(f'EP {i} simulation finished, start learning')
            q_new = self.update_q(q, self.pool ,i, 0.95, lr)
            self.q = q_new
            print(np.mean(r_ls))
            self.qs.append(q_new)
            print(f'EP {i} learning finished: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
            if q_new.difference(q).L_1() < threshold and False:
                break
            else:
                q = q_new
        return q_new
    
    def update_q(self, q, pool, t ,alpha, lr):
        pool = pool.shuffle()
        for _ in range(1):
            q = deepcopy(q)
#           
            for s, next_s, a, r, is_random in pool:
#                 if is_random: continue
                q[s, a] = q[s,a] + lr(t, s, a)*(r + alpha*q.max_value_act(next_s)[0]-q[s,a])
#             print(f'Learning ep: {i}: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
#             if q_new.difference(q).L_1() < self.threshold:
#                 break
#             else:
#                 q = q_new
        return q
```


```python
# Q-learning with epsilon-greedy policy,  learning rate = 1/t
q = Q_table(random = False)
env = Env()
q_one_over_t = Q_learning(q, env, MAX_ITER = 200)
lr_fn = lambda i, s, a: 1/(i+1)
lr_fn = lambda i, s, a: 0.1
class Learning_Rate:
    def __init__(self, state_space, control_space):
        self.counter = dict(
                zip(list(product(state_space, control_space)), [0]*len(state_space)*len(control_space))
        )
    def __call__(self, i, s, a):
        self.counter[(s, a)] += 1
        return 1/self.counter[(s, a)]
lr_fn = Learning_Rate(state_space, control_space)
learned_q = q_one_over_t.train(lr_fn)
```

    0
    1
    2
    3
    EP 3 simulation finished, start learning
    -2.8
    EP 3 learning finished: 10.0, 1.203125
    4
    EP 4 simulation finished, start learning
    -1.0
    EP 4 learning finished: 4.0375, 0.2749488932291667
    5
    EP 5 simulation finished, start learning
    -3.0
    EP 5 learning finished: 3.7388888888888894, 0.25877616139846743
    6
    EP 6 simulation finished, start learning
    -2.8
    EP 6 learning finished: 3.0, 0.1178298611111111
    7
    EP 7 simulation finished, start learning
    -3.0
    EP 7 learning finished: 3.0, 0.10276620370370368
    8
    EP 8 simulation finished, start learning
    -2.8
    EP 8 learning finished: 3.0, 0.12802372685185184
    9
    EP 9 simulation finished, start learning
    -3.0
    EP 9 learning finished: 3.0, 0.09375
    10
    EP 10 simulation finished, start learning
    0.0
    EP 10 learning finished: 3.7916666666666665, 0.1707088551387081
    11
    EP 11 simulation finished, start learning
    1.2
    EP 11 learning finished: 1.95, 0.17564755201498225
    12
    EP 12 simulation finished, start learning
    2.1
    EP 12 learning finished: 8.179383011623111, 0.36755039006794205
    13
    EP 13 simulation finished, start learning
    2.3
    EP 13 learning finished: 7.180949097635194, 0.505502544413277
    14
    EP 14 simulation finished, start learning
    1.0
    EP 14 learning finished: 10.529638094503795, 0.6687878933050688
    15
    EP 15 simulation finished, start learning
    1.2
    EP 15 learning finished: 1.5832836084793396, 0.19419054627903892
    16
    EP 16 simulation finished, start learning
    1.0
    EP 16 learning finished: 2.511370648499871, 0.2057422100350488
    17
    EP 17 simulation finished, start learning
    2.3
    EP 17 learning finished: 0.7322650383760363, 0.10775942511788877
    18
    EP 18 simulation finished, start learning
    1.2
    EP 18 learning finished: 0.758079236315659, 0.08259328993753481
    19
    EP 19 simulation finished, start learning
    1.2
    EP 19 learning finished: 0.5445643322012454, 0.06349375097195324
    20
    EP 20 simulation finished, start learning
    3.4
    EP 20 learning finished: 0.2910884590607967, 0.04585147107954693
    21
    EP 21 simulation finished, start learning
    2.3
    EP 21 learning finished: 0.20711559336460894, 0.03815604019716601
    22
    EP 22 simulation finished, start learning
    2.3
    EP 22 learning finished: 0.24045311912940548, 0.03306653546216029
    23
    EP 23 simulation finished, start learning
    1.2
    EP 23 learning finished: 11.32467153775542, 0.3856745305834582
    24
    EP 24 simulation finished, start learning
    1.2
    EP 24 learning finished: 0.4694465320162551, 0.03329169129853242
    25
    EP 25 simulation finished, start learning
    1.2
    EP 25 learning finished: 0.313332353557755, 0.03147756869478076
    26
    EP 26 simulation finished, start learning
    3.4
    EP 26 learning finished: 0.367177907022072, 0.029835467382556935
    27
    EP 27 simulation finished, start learning
    1.2
    EP 27 learning finished: 0.24167800293405328, 0.031215206446343527
    28
    EP 28 simulation finished, start learning
    -0.1
    EP 28 learning finished: 0.21946496652876402, 0.0457360283860693
    29
    EP 29 simulation finished, start learning
    3.4
    EP 29 learning finished: 0.24136972965107972, 0.04124583518176052
    30
    EP 30 simulation finished, start learning
    2.1
    EP 30 learning finished: 9.889141207075749, 0.3467720088938498
    31
    EP 31 simulation finished, start learning
    1.0
    EP 31 learning finished: 0.37442228929999555, 0.04653919586224753
    32
    EP 32 simulation finished, start learning
    1.2
    EP 32 learning finished: 0.26468185846133396, 0.0278454489516963
    33
    EP 33 simulation finished, start learning
    1.2
    EP 33 learning finished: 1.6276195506074815, 0.0739568285734516
    34
    EP 34 simulation finished, start learning
    1.2
    EP 34 learning finished: 1.1006663375986974, 0.0538595788169145
    35
    EP 35 simulation finished, start learning
    1.2
    EP 35 learning finished: 0.7937284253335761, 0.03905421702459888
    36
    EP 36 simulation finished, start learning
    2.3
    EP 36 learning finished: 0.6084186896515882, 0.03276660482033396
    37
    EP 37 simulation finished, start learning
    0.1
    EP 37 learning finished: 0.0943330639347888, 0.017666022268983284
    38
    EP 38 simulation finished, start learning
    0.8
    EP 38 learning finished: 0.5918245931472773, 0.03730792738356149
    39
    EP 39 simulation finished, start learning
    2.3
    EP 39 learning finished: 0.5220786428057114, 0.03302973821376948
    40
    EP 40 simulation finished, start learning
    2.1
    EP 40 learning finished: 0.516223788970045, 0.04714246591027125
    41
    EP 41 simulation finished, start learning
    1.2
    EP 41 learning finished: 0.41609239372537415, 0.038109917498024316
    42
    EP 42 simulation finished, start learning
    2.3
    EP 42 learning finished: 0.27550275516550116, 0.021713991523740594
    43
    EP 43 simulation finished, start learning
    2.3
    EP 43 learning finished: 0.21502786032986343, 0.020953646707829565
    44
    EP 44 simulation finished, start learning
    2.3
    EP 44 learning finished: 0.10336802419824398, 0.013689832251371437
    45
    EP 45 simulation finished, start learning
    -0.1
    EP 45 learning finished: 14.179810448394306, 0.45858777580326654
    46
    EP 46 simulation finished, start learning
    2.3
    EP 46 learning finished: 0.15651116189921677, 0.015660649504307483
    47
    EP 47 simulation finished, start learning
    1.0
    EP 47 learning finished: 0.19041621386189078, 0.01926685882810708
    48
    EP 48 simulation finished, start learning
    1.2
    EP 48 learning finished: 0.1554773102524738, 0.01765675700603156
    49
    EP 49 simulation finished, start learning
    1.2
    EP 49 learning finished: 0.12769688703946436, 0.01434993231892892
    50
    EP 50 simulation finished, start learning
    1.0
    EP 50 learning finished: 0.22472316047797047, 0.021542081261902685
    51
    EP 51 simulation finished, start learning
    1.2
    EP 51 learning finished: 0.2136603206211407, 0.01695516746657759
    52
    EP 52 simulation finished, start learning
    0.1
    EP 52 learning finished: 1.0518054405047867, 0.05226095683454943
    53
    EP 53 simulation finished, start learning
    1.0
    EP 53 learning finished: 0.8030643755542826, 0.05107481822106406
    54
    EP 54 simulation finished, start learning
    2.3
    EP 54 learning finished: 0.6322686961251904, 0.03810580529791915
    55
    EP 55 simulation finished, start learning
    2.3
    EP 55 learning finished: 0.5150570440142812, 0.0558974904437355
    56
    EP 56 simulation finished, start learning
    2.3
    EP 56 learning finished: 0.7517025267376982, 0.04413887620086203
    57
    EP 57 simulation finished, start learning
    2.1
    EP 57 learning finished: 0.5235807088045785, 0.03533485566364636
    58
    EP 58 simulation finished, start learning
    2.3
    EP 58 learning finished: 0.3875610343285265, 0.029696596261314472
    59
    EP 59 simulation finished, start learning
    0.8
    EP 59 learning finished: 0.1963220069383258, 0.030084134395540262
    60
    EP 60 simulation finished, start learning
    -0.1
    EP 60 learning finished: 0.19316879721820435, 0.026791454254621986
    61
    EP 61 simulation finished, start learning
    -0.1
    EP 61 learning finished: 1.0757151604193753, 0.05669243858186339
    62
    EP 62 simulation finished, start learning
    2.3
    EP 62 learning finished: 0.7302163245721838, 0.04461309152677423
    63
    EP 63 simulation finished, start learning
    -0.1
    EP 63 learning finished: 10.441933945133925, 0.36670547829692424
    64
    EP 64 simulation finished, start learning
    2.3
    EP 64 learning finished: 0.4148024709991736, 0.03221509976105633
    65
    EP 65 simulation finished, start learning
    2.3
    EP 65 learning finished: 0.23019302213500792, 0.018256908419465714
    66
    EP 66 simulation finished, start learning
    1.0
    EP 66 learning finished: 0.2144774572016761, 0.018984554699339434
    67
    EP 67 simulation finished, start learning
    1.2
    EP 67 learning finished: 0.08843543290554834, 0.008547286240470409
    68
    EP 68 simulation finished, start learning
    1.0
    EP 68 learning finished: 0.3291532499854223, 0.01851264690470014
    69
    EP 69 simulation finished, start learning
    1.2
    EP 69 learning finished: 0.30841188009335774, 0.018011632384037413
    70
    EP 70 simulation finished, start learning
    1.2
    EP 70 learning finished: 0.2900719349274485, 0.017637786143040346
    71
    EP 71 simulation finished, start learning
    1.2
    EP 71 learning finished: 0.2735233418939113, 0.022469551500246626
    72
    EP 72 simulation finished, start learning
    2.1
    EP 72 learning finished: 0.17974625043041748, 0.013847753997237644
    73
    EP 73 simulation finished, start learning
    0.1
    EP 73 learning finished: 0.7142533450471134, 0.04232670343088525
    74
    EP 74 simulation finished, start learning
    0.1
    EP 74 learning finished: 0.5800361584151176, 0.04317535311641961
    75
    EP 75 simulation finished, start learning
    1.2
    EP 75 learning finished: 0.4775897792446351, 0.033528612149326203
    76
    EP 76 simulation finished, start learning
    1.2
    EP 76 learning finished: 0.404616435647563, 0.02888777724521325
    77
    EP 77 simulation finished, start learning
    2.3
    EP 77 learning finished: 0.13828358450894562, 0.014521023118479248
    78
    EP 78 simulation finished, start learning
    1.2
    EP 78 learning finished: 0.09861068288603114, 0.009738532443922254
    79
    EP 79 simulation finished, start learning
    2.1
    EP 79 learning finished: 0.17192469221424256, 0.014535184802963438
    80
    EP 80 simulation finished, start learning
    1.2
    EP 80 learning finished: 0.16462170066512472, 0.014415460307259438
    81
    EP 81 simulation finished, start learning
    -0.1
    EP 81 learning finished: 0.35449446699330533, 0.028244427787153685
    82
    EP 82 simulation finished, start learning
    2.3
    EP 82 learning finished: 0.3075114352467283, 0.026403192002443652
    83
    EP 83 simulation finished, start learning
    1.2
    EP 83 learning finished: 0.2697379631411958, 0.023733162407592753
    84
    EP 84 simulation finished, start learning
    1.2
    EP 84 learning finished: 0.23747190429430276, 0.021584135418997938
    85
    EP 85 simulation finished, start learning
    1.0
    EP 85 learning finished: 0.16653378519199968, 0.013228224018870471
    86
    EP 86 simulation finished, start learning
    0.1
    EP 86 learning finished: 0.14436677813993626, 0.012234372754152067
    87
    EP 87 simulation finished, start learning
    2.1
    EP 87 learning finished: 0.26560376618521403, 0.017562428271867284
    88
    EP 88 simulation finished, start learning
    1.0
    EP 88 learning finished: 0.4336764529210846, 0.037957600916937444
    89
    EP 89 simulation finished, start learning
    0.1
    EP 89 learning finished: 0.3906893385793475, 0.04047259920659939
    90
    EP 90 simulation finished, start learning
    1.2
    EP 90 learning finished: 0.3548988486551492, 0.03569444870953762
    91
    EP 91 simulation finished, start learning
    1.0
    EP 91 learning finished: 0.3236208451400142, 0.028527579515954894
    92
    EP 92 simulation finished, start learning
    1.0
    EP 92 learning finished: 1.3792829646295957, 0.056000871773252414
    93
    EP 93 simulation finished, start learning
    2.3
    EP 93 learning finished: 0.927857900946732, 0.03572490815978999
    94
    EP 94 simulation finished, start learning
    2.1
    EP 94 learning finished: 0.6669014549722698, 0.02935373551125714
    95
    EP 95 simulation finished, start learning
    1.2
    EP 95 learning finished: 0.5037147055945788, 0.022047425669598664
    96
    EP 96 simulation finished, start learning
    2.3
    EP 96 learning finished: 0.05526282135850025, 0.005897457872729467
    97
    EP 97 simulation finished, start learning
    1.2
    EP 97 learning finished: 0.08394821563888755, 0.008144209235856792
    98
    EP 98 simulation finished, start learning
    1.9
    EP 98 learning finished: 0.207507437936151, 0.01413172453793149
    99
    EP 99 simulation finished, start learning
    -0.3
    EP 99 learning finished: 0.29937281795662063, 0.026911957673027842
    100
    EP 100 simulation finished, start learning
    2.1
    EP 100 learning finished: 0.2755498001242751, 0.026301999184322654
    101
    EP 101 simulation finished, start learning
    2.3
    EP 101 learning finished: 0.2539054374616092, 0.027929096278668717
    102
    EP 102 simulation finished, start learning
    2.3
    EP 102 learning finished: 0.23462874136780743, 0.021757427579759826
    103
    EP 103 simulation finished, start learning
    -0.1
    EP 103 learning finished: 0.1390780962937096, 0.01581193287032329
    104
    EP 104 simulation finished, start learning
    -0.1
    EP 104 learning finished: 0.21856654808952092, 0.02281169063204986
    105
    EP 105 simulation finished, start learning
    1.0
    EP 105 learning finished: 0.2334004333463211, 0.024613288509079803
    106
    EP 106 simulation finished, start learning
    -0.3
    EP 106 learning finished: 1.9693461311664944, 0.08581614864629653
    107
    EP 107 simulation finished, start learning
    -0.3
    EP 107 learning finished: 1.4103134582349235, 0.06866602846532566
    108
    EP 108 simulation finished, start learning
    1.0
    EP 108 learning finished: 1.0592972159054257, 0.050549977794041516
    109
    EP 109 simulation finished, start learning
    2.1
    EP 109 learning finished: 0.8256108925761403, 0.0412711371787777
    110
    EP 110 simulation finished, start learning
    1.2
    EP 110 learning finished: 0.15714972854154663, 0.013567064005796131
    111
    EP 111 simulation finished, start learning
    2.3
    EP 111 learning finished: 0.14789352057618022, 0.009247815322174169
    112
    EP 112 simulation finished, start learning
    1.9
    EP 112 learning finished: 0.13962223544778674, 0.009315736526419482
    113
    EP 113 simulation finished, start learning
    1.2
    EP 113 learning finished: 0.03397908108655656, 0.0052244440882305565
    114
    EP 114 simulation finished, start learning
    3.4
    EP 114 learning finished: 0.03996446520095631, 0.005622172099377654
    115
    EP 115 simulation finished, start learning
    1.2
    EP 115 learning finished: 0.03289378889368422, 0.005610393240796263
    116
    EP 116 simulation finished, start learning
    2.3
    EP 116 learning finished: 0.023587429474679844, 0.004021303608557042
    117
    EP 117 simulation finished, start learning
    1.0
    EP 117 learning finished: 0.046320937553014474, 0.005056449576141597
    118
    EP 118 simulation finished, start learning
    2.1
    EP 118 learning finished: 0.0834577543267665, 0.005836581197552704
    119
    EP 119 simulation finished, start learning
    1.2
    EP 119 learning finished: 0.07337689830120908, 0.0072883067085472986
    120
    EP 120 simulation finished, start learning
    2.1
    EP 120 learning finished: 0.4284676154401908, 0.020141979633204937
    121
    EP 121 simulation finished, start learning
    1.2
    EP 121 learning finished: 0.3442442264489202, 0.016418007645489938
    122
    EP 122 simulation finished, start learning
    1.2
    EP 122 learning finished: 0.2836374186352356, 0.013339866532737114
    123
    EP 123 simulation finished, start learning
    1.2
    EP 123 learning finished: 0.2369613750477707, 0.010325601626695827
    124
    EP 124 simulation finished, start learning
    3.0
    EP 124 learning finished: 0.05095337192672389, 0.004707889468924281
    125
    EP 125 simulation finished, start learning
    1.0
    EP 125 learning finished: 0.13604626730418712, 0.00877728685337642
    126
    EP 126 simulation finished, start learning
    2.3
    EP 126 learning finished: 0.12899346724185623, 0.008525447180251278
    127
    EP 127 simulation finished, start learning
    -0.3
    EP 127 learning finished: 0.14465629880650077, 0.012675039272568717
    128
    EP 128 simulation finished, start learning
    2.3
    EP 128 learning finished: 0.1165795570056769, 0.010033362277775376
    129
    EP 129 simulation finished, start learning
    2.1
    EP 129 learning finished: 0.09710603246040073, 0.006622117435166552
    130
    EP 130 simulation finished, start learning
    1.0
    EP 130 learning finished: 0.11037193487590091, 0.009662780588792219
    131
    EP 131 simulation finished, start learning
    -0.1
    EP 131 learning finished: 0.1872363594848072, 0.014217576083615804
    132
    EP 132 simulation finished, start learning
    2.3
    EP 132 learning finished: 0.17622245598570108, 0.013605925673584852
    133
    EP 133 simulation finished, start learning
    1.2
    EP 133 learning finished: 0.166715120498534, 0.013119214124687535
    134
    EP 134 simulation finished, start learning
    1.0
    EP 134 learning finished: 0.2575973501449582, 0.017747029471895756
    135
    EP 135 simulation finished, start learning
    1.2
    EP 135 learning finished: 0.24711199243829118, 0.012674635078504921
    136
    EP 136 simulation finished, start learning
    2.3
    EP 136 learning finished: 0.23674365059797275, 0.012183095897623154
    137
    EP 137 simulation finished, start learning
    1.2
    EP 137 learning finished: 0.2271657038625623, 0.012587359466021686
    138
    EP 138 simulation finished, start learning
    1.2
    EP 138 learning finished: 0.2486511813604142, 0.01294959864698491
    139
    EP 139 simulation finished, start learning
    2.3
    EP 139 learning finished: 0.22277912523892152, 0.010772762321571372
    140
    EP 140 simulation finished, start learning
    1.2
    EP 140 learning finished: 0.19997712663702538, 0.010132711426246208
    141
    EP 141 simulation finished, start learning
    1.0
    EP 141 learning finished: 0.18148289508968318, 0.010847277967876795
    142
    EP 142 simulation finished, start learning
    1.0
    EP 142 learning finished: 0.21868324737543832, 0.019119865255533552
    143
    EP 143 simulation finished, start learning
    2.3
    EP 143 learning finished: 0.2874969975737507, 0.022616809382695276
    144
    EP 144 simulation finished, start learning
    1.2
    EP 144 learning finished: 0.2437683794118093, 0.021396673638952382
    145
    EP 145 simulation finished, start learning
    1.2
    EP 145 learning finished: 0.21067877487917253, 0.01813768392013588
    146
    EP 146 simulation finished, start learning
    1.0
    EP 146 learning finished: 0.2511918745314188, 0.01625546853881625
    147
    EP 147 simulation finished, start learning
    1.0
    EP 147 learning finished: 0.23963681701475537, 0.013443818062296803
    148
    EP 148 simulation finished, start learning
    2.3
    EP 148 learning finished: 0.228725868056471, 0.013681528239246427
    149
    EP 149 simulation finished, start learning
    1.2
    EP 149 learning finished: 0.2187799851995922, 0.014313302360662347
    150
    EP 150 simulation finished, start learning
    1.0
    EP 150 learning finished: 0.0677446988277115, 0.00599282840989801
    151
    EP 151 simulation finished, start learning
    2.3
    EP 151 learning finished: 0.062422696163304536, 0.005998317586763835
    152
    EP 152 simulation finished, start learning
    1.2
    EP 152 learning finished: 0.029454282874873172, 0.004769535590375795
    153
    EP 153 simulation finished, start learning
    1.2
    EP 153 learning finished: 0.02689470424297724, 0.00426677777214271
    154
    EP 154 simulation finished, start learning
    2.3
    EP 154 learning finished: 0.025928202677853562, 0.0037593530809027498
    155
    EP 155 simulation finished, start learning
    1.0
    EP 155 learning finished: 0.027042126004051426, 0.0036929447933346515
    156
    EP 156 simulation finished, start learning
    1.0
    EP 156 learning finished: 0.1452930672885535, 0.00816338486072006
    157
    EP 157 simulation finished, start learning
    2.3
    EP 157 learning finished: 0.1383743497986225, 0.007633079146310749
    158
    EP 158 simulation finished, start learning
    -0.1
    EP 158 learning finished: 0.23510060398373578, 0.015540904782526739
    159
    EP 159 simulation finished, start learning
    2.3
    EP 159 learning finished: 0.22340276235916567, 0.014126211053712041
    160
    EP 160 simulation finished, start learning
    1.2
    EP 160 learning finished: 0.21243051239636968, 0.00992822092895819
    161
    EP 161 simulation finished, start learning
    1.2
    EP 161 learning finished: 0.20244911415159783, 0.010073671768405588
    162
    EP 162 simulation finished, start learning
    2.1
    EP 162 learning finished: 0.12081599017608369, 0.006635958007569304
    163
    EP 163 simulation finished, start learning
    2.3
    EP 163 learning finished: 0.11568098520564485, 0.00666323290684763
    164
    EP 164 simulation finished, start learning
    -0.3
    EP 164 learning finished: 0.2176826889696143, 0.01756025301949737
    165
    EP 165 simulation finished, start learning
    2.1
    EP 165 learning finished: 0.2011097476392878, 0.018269647493330914
    166
    EP 166 simulation finished, start learning
    1.2
    EP 166 learning finished: 0.1769312083312035, 0.014719378629555458
    167
    EP 167 simulation finished, start learning
    2.3
    EP 167 learning finished: 0.17077703196512473, 0.014019646550337403
    168
    EP 168 simulation finished, start learning
    0.8
    EP 168 learning finished: 0.2139781554024145, 0.012007394500387847
    169
    EP 169 simulation finished, start learning
    1.2
    EP 169 learning finished: 0.20490000135285946, 0.010373831709034803
    170
    EP 170 simulation finished, start learning
    1.2
    EP 170 learning finished: 0.19618085235912064, 0.01005857413844359
    171
    EP 171 simulation finished, start learning
    2.3
    EP 171 learning finished: 0.18816805150782523, 0.009896310162120572
    172
    EP 172 simulation finished, start learning
    1.2
    EP 172 learning finished: 0.028318162146648618, 0.0036484842451834643
    173
    EP 173 simulation finished, start learning
    3.4
    EP 173 learning finished: 0.02724157603927857, 0.003255195836668584
    174
    EP 174 simulation finished, start learning
    1.0
    EP 174 learning finished: 0.19517379823403513, 0.009137230267717851
    175
    EP 175 simulation finished, start learning
    2.1
    EP 175 learning finished: 0.1863046639270889, 0.01185410008644227
    176
    EP 176 simulation finished, start learning
    2.3
    EP 176 learning finished: 0.177998630730555, 0.010889026527701698
    177
    EP 177 simulation finished, start learning
    2.3
    EP 177 learning finished: 0.17020590709032524, 0.010273199854781798
    178
    EP 178 simulation finished, start learning
    1.2
    EP 178 learning finished: 0.08809912882122806, 0.00564015592014655
    179
    EP 179 simulation finished, start learning
    1.2
    EP 179 learning finished: 0.021674852928583732, 0.0028938198366477152
    180
    EP 180 simulation finished, start learning
    2.3
    EP 180 learning finished: 0.040312195440669996, 0.003214619028915233
    181
    EP 181 simulation finished, start learning
    1.0
    EP 181 learning finished: 0.08487175698675209, 0.0057434712743787975
    182
    EP 182 simulation finished, start learning
    0.8
    EP 182 learning finished: 0.08162262849635482, 0.006895134228235539
    183
    EP 183 simulation finished, start learning
    1.2
    EP 183 learning finished: 0.09747489819060107, 0.009653425854387138
    184
    EP 184 simulation finished, start learning
    -0.1
    EP 184 learning finished: 0.14797868529173241, 0.01511395925596315
    185
    EP 185 simulation finished, start learning
    1.2
    EP 185 learning finished: 0.14042050259096928, 0.012117237066619983
    186
    EP 186 simulation finished, start learning
    2.1
    EP 186 learning finished: 0.13355179972305287, 0.010916416023957057
    187
    EP 187 simulation finished, start learning
    1.2
    EP 187 learning finished: 0.12708788905382717, 0.008080413524953527
    188
    EP 188 simulation finished, start learning
    2.3
    EP 188 learning finished: 0.021334617252994903, 0.0029575046516102377
    189
    EP 189 simulation finished, start learning
    1.2
    EP 189 learning finished: 0.020384705971707717, 0.002859683454135953
    190
    EP 190 simulation finished, start learning
    2.3
    EP 190 learning finished: 0.01720991513113912, 0.0025096634951575214
    191
    EP 191 simulation finished, start learning
    1.2
    EP 191 learning finished: 0.016942165340910975, 0.0023667351123916047
    192
    EP 192 simulation finished, start learning
    2.3
    EP 192 learning finished: 0.015544415403009992, 0.0022781518665354883
    193
    EP 193 simulation finished, start learning
    1.2
    EP 193 learning finished: 0.015065967454120255, 0.0021413052331463733
    194
    EP 194 simulation finished, start learning
    2.3
    EP 194 learning finished: 0.02738969109633871, 0.0032253032369653767
    195
    EP 195 simulation finished, start learning
    2.1
    EP 195 learning finished: 1.0548760383333087, 0.03602714726509955
    196
    EP 196 simulation finished, start learning
    2.3
    EP 196 learning finished: 0.7921247896486479, 0.027672478520809528
    197
    EP 197 simulation finished, start learning
    2.3
    EP 197 learning finished: 0.616950358706784, 0.022151659316265216
    198
    EP 198 simulation finished, start learning
    1.2
    EP 198 learning finished: 0.49432210967091783, 0.017042049193065532
    199
    EP 199 simulation finished, start learning
    1.2
    EP 199 learning finished: 0.02390093982551278, 0.0024459456905122356



```python
q_one_over_t.q
```




              0          1          2          3          4          5          6  \
    0 -0.318109   4.009127   6.372237  17.037176  20.494916   2.475021   1.334324   
    1  8.009818  -2.290960   0.553147  12.218198  14.035644   2.651693  -0.819693   
    2 -1.525738  -1.685938   6.343837  10.801964  15.428148  13.830476  10.858367   
    3 -0.237116  10.887545  13.868530  11.426350  15.117853   6.665946   5.992968   
    
              7  
    0 -0.041727  
    1  7.808959  
    2 -3.897222  
    3  1.058568  




```python
v_l, pi_l= q_one_over_t.q.yield_v_pi()
```


```python
v_l
```




    0     8.009818
    1    10.887545
    2    13.868530
    3    17.037176
    4    20.494916
    5    13.830476
    6    10.858367
    7     7.808959
    dtype: float64




```python
display_pi(pi_l)
```

    
        -----   -----   -----
        | d |   | u |   | d |
        ---------------------
        | r | r | u | l | l |
        ---------------------
        



```python
# 1 action delay, 0 obs. delay

class delayed_wrapper:
    def __init__(self, env, delay_steps):
        self.env = env
        self.num = delay_steps
        self.state_dim = env.state_dim
        self.action_dim = env.action_dim
        self.action_buffer = [-1]*delay_steps
        
    def step(self, action):
        app_action = self.action_buffer.pop(0)
        self.action_buffer.append(action)
        return  self.env.step(app_action)
    
    def reset(self, random = False):
        self.action_buffer = [0]*self.num
        return self.env.reset(random)
    

```


```python
np.random.choice(control_space, 10)
```




    array([2, 2, 0, 3, 0, 1, 2, 0, 1, 0])




```python
# Q-learning using model, async reward collection


class Q_learning:
    def __init__(self, q, env, delay_step, model, epsilon = 0.1, MAX_ITER = 10000, episode_length = 100, threshold = 1e-3):
        self.q = q
        self.env = env
        self.delay_step = delay_step
        self.epsilon = epsilon
        self.MAX_ITER = MAX_ITER
        self.episode_length = episode_length
        self.threshold = threshold
        self.pool = Pool(40)
        self.qs = []
        self.action_memory = list(np.random.choice(control_space, delay_step))
        self.model = model
        
    def train(self, lr = lambda *args: 0.1):
        q = self.q
        env = self.env
        epsilon = self.epsilon
        MAX_ITER = self.MAX_ITER
        episode_length = self.episode_length
        threshold = self.threshold
        model = self.model
        for i in range(MAX_ITER):
            print(i)
            obs = env.reset(random = True)
            r_ls = []
            for _ in range(10):
                
                est_obs = model(obs, self.action_memory)
                last_action = self.action_memory.pop(0)
                action, is_random = (np.random.choice(control_space), True) \
                                    if np.random.uniform() < epsilon else \
                                    (q.max_value_act(est_obs)[1], False)
                self.action_memory.append(action)
                next_obs, reward, _, _ = env.step(action)
                self.pool.append((obs, next_obs, last_action, reward, is_random))
                r_ls.append(reward)
                obs = next_obs
            if not self.pool.isFull:
                continue
            print(f'EP {i} simulation finished, start learning')
            q_new = self.update_q(q, self.pool ,i, 0.95, lr)
            self.q = q_new
            print(np.mean(r_ls))
            self.qs.append(q_new)
            print(f'EP {i} learning finished: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
            if q_new.difference(q).L_1() < threshold and False:
                break
            else:
                q = q_new
        return q_new
    
    def update_q(self, q, pool, t ,alpha, lr):
        pool = pool.shuffle()
        for _ in range(1):
            q = deepcopy(q)
#           
            for s, next_s, a, r, is_random in pool:
#                 if is_random: continue
                q[s, a] = q[s,a] + lr(t, s, a)*(r + alpha*q.max_value_act(next_s)[0]-q[s,a])
#             print(f'Learning ep: {i}: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
#             if q_new.difference(q).L_1() < self.threshold:
#                 break
#             else:
#                 q = q_new
        return q
```


```python
def generate_trans_matrix(state_space, action_space, graph):
    ret = np.zeros(shape = (len(action_space), len(state_space), len(state_space)))
    for a in action_space:
        for s in state_space:
            for ns in state_space:
                if graph[s][a] < 0:
                    ret[a, s, ns] = 1. if s == ns else 0.
                elif graph[s][a] <= 10:
                    ret[a, s, ns] = 1. if ns == graph[s][a] else 0.
                else:
                    ret[a, s, ns] = 0.125
    return ret

trans_matrix = generate_trans_matrix(state_space, control_space, graph)
trans_matrix
```




    array([[[1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   ],
            [0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ]],
    
           [[0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ]],
    
           [[1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ]],
    
           [[1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ]]])




```python
def generate_trans_matrix(state_space, action_space, graph):
    ret = np.zeros(shape = (len(action_space), len(state_space), len(state_space)))
    for a in action_space:
        for s in state_space:
            for ns in state_space:
                if graph[s][a] < 0:
                    ret[a, s, ns] = 1. if ns == s else 0.
                elif graph[s][a] <= 10:
                    ret[a, s, ns] = 1. if ns == graph[s][a] else 0.
                else:
                    ret[a, s, ns] = 0.125
    return ret

trans_matrix = generate_trans_matrix(state_space, control_space, graph)
trans_matrix
```




    array([[[1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   ],
            [0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ]],
    
           [[0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ]],
    
           [[1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ]],
    
           [[1.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 1.   , 0.   , 0.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   , 0.   ],
            [0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 0.   , 1.   ]]])




```python
mu = np.argmax(trans_matrix, axis = 2)
print(mu)
def model_model(x, u):
    '''
    x: newest observed state
    u: a list of actions in queue
    '''
    assert type(u)==list
    u = u[0]
    try:
        m = mu[u]
    except:
        raise ValueError
    return m[x]
```

    [[0 0 2 4 0 5 7 7]
     [1 1 2 3 3 5 6 6]
     [0 1 1 2 4 3 5 7]
     [0 2 3 5 4 6 6 7]]



```python
class Learning_Rate:
    def __init__(self, state_space, control_space):
        self.counter = dict(
                zip(list(product(state_space, control_space)), [0]*len(state_space)*len(control_space))
        )
    def __call__(self, i, s, a):
        self.counter[(s, a)] += 1
        return 1/self.counter[(s, a)]
lr_fn = Learning_Rate(state_space, control_space)
# lr_fn = lambda i,s,a: 1/(i+1)
# lr_fn = lambda i,s,a: 0.1
env = delayed_wrapper(Env(), 1)
q = Q_table(random = False)
q_model = Q_learning(q, env, 1, model_model, MAX_ITER = 1000)
learned_q = q_model.train(lr_fn)
```

    0
    1
    2
    3
    EP 3 simulation finished, start learning
    -3.0
    EP 3 learning finished: 16.721249999999998, 1.6379910714285715
    4
    EP 4 simulation finished, start learning
    -1.4
    EP 4 learning finished: 3.1925937499999986, 0.22227971540178565
    5
    EP 5 simulation finished, start learning
    -1.7
    EP 5 learning finished: 10.0, 0.40625
    6
    EP 6 simulation finished, start learning
    0.4
    EP 6 learning finished: 13.288890624999997, 1.110075960286458
    7
    EP 7 simulation finished, start learning
    0.6
    EP 7 learning finished: 9.268405272493487, 0.5818214050281341
    8
    EP 8 simulation finished, start learning
    -3.0
    EP 8 learning finished: 3.83125, 0.20187696850246745
    9
    EP 9 simulation finished, start learning
    -2.6
    EP 9 learning finished: 3.0, 0.21944869375113538
    10
    EP 10 simulation finished, start learning
    -2.0
    EP 10 learning finished: 2.666666666666667, 0.16016307307584338
    11
    EP 11 simulation finished, start learning
    -2.8
    EP 11 learning finished: 3.83125, 0.14842264261776145
    12
    EP 12 simulation finished, start learning
    -2.8
    EP 12 learning finished: 3.0, 0.11233001060917824
    13
    EP 13 simulation finished, start learning
    0.8
    EP 13 learning finished: 9.236620839372854, 0.432044696496267
    14
    EP 14 simulation finished, start learning
    0.8
    EP 14 learning finished: 2.2981592326534837, 0.191179130048992
    15
    EP 15 simulation finished, start learning
    -0.1
    EP 15 learning finished: 1.4653070597514943, 0.14646439578254417
    16
    EP 16 simulation finished, start learning
    -0.1
    EP 16 learning finished: 1.1098650943373287, 0.09635907976749603
    17
    EP 17 simulation finished, start learning
    -0.5
    EP 17 learning finished: 12.264623701192287, 0.518450277802621
    18
    EP 18 simulation finished, start learning
    -0.3
    EP 18 learning finished: 1.0356793347673487, 0.12586614325111029
    19
    EP 19 simulation finished, start learning
    0.6
    EP 19 learning finished: 2.3625439004352895, 0.2521969155093341
    20
    EP 20 simulation finished, start learning
    -0.3
    EP 20 learning finished: 2.7101866831875134, 0.21307371255054175
    21
    EP 21 simulation finished, start learning
    0.6
    EP 21 learning finished: 1.507436886448569, 0.15178807823709278
    22
    EP 22 simulation finished, start learning
    0.8
    EP 22 learning finished: 10.865229962970211, 0.7057023880179587
    23
    EP 23 simulation finished, start learning
    -0.7
    EP 23 learning finished: 0.512152526108189, 0.08547203440418216
    24
    EP 24 simulation finished, start learning
    -0.1
    EP 24 learning finished: 0.4228703491026393, 0.07358121060671319
    25
    EP 25 simulation finished, start learning
    0.1
    EP 25 learning finished: 0.6621230989057194, 0.07858414866549931
    26
    EP 26 simulation finished, start learning
    1.0
    EP 26 learning finished: 0.9667183092651337, 0.10360049954212908
    27
    EP 27 simulation finished, start learning
    0.6
    EP 27 learning finished: 0.6702024559288091, 0.08075088566462063
    28
    EP 28 simulation finished, start learning
    0.8
    EP 28 learning finished: 0.493110321099814, 0.06955662494830947
    29
    EP 29 simulation finished, start learning
    1.2
    EP 29 learning finished: 0.703035566666836, 0.06863515828301767
    30
    EP 30 simulation finished, start learning
    -0.3
    EP 30 learning finished: 0.5614240510054014, 0.050195500868880694
    31
    EP 31 simulation finished, start learning
    -0.1
    EP 31 learning finished: 0.47356373947920183, 0.05724239158138462
    32
    EP 32 simulation finished, start learning
    0.8
    EP 32 learning finished: 0.360882009613956, 0.05364597284364961
    33
    EP 33 simulation finished, start learning
    -0.3
    EP 33 learning finished: 0.2962504869881344, 0.04913771669750964
    34
    EP 34 simulation finished, start learning
    1.0
    EP 34 learning finished: 0.24127104996121673, 0.05929436919368136
    35
    EP 35 simulation finished, start learning
    -0.5
    EP 35 learning finished: 0.8240646790044068, 0.07615510225699999
    36
    EP 36 simulation finished, start learning
    -0.3
    EP 36 learning finished: 0.5996773840055398, 0.07452565708366361
    37
    EP 37 simulation finished, start learning
    1.9
    EP 37 learning finished: 0.6868254518599954, 0.08897747933787695
    38
    EP 38 simulation finished, start learning
    0.8
    EP 38 learning finished: 9.164735700910166, 0.35210382036753507
    39
    EP 39 simulation finished, start learning
    -0.5
    EP 39 learning finished: 0.3280940516574482, 0.04374082190964795
    40
    EP 40 simulation finished, start learning
    -0.3
    EP 40 learning finished: 0.25136403634927085, 0.040767322845044515
    41
    EP 41 simulation finished, start learning
    0.8
    EP 41 learning finished: 1.2407626800849663, 0.07076418744759844
    42
    EP 42 simulation finished, start learning
    1.0
    EP 42 learning finished: 0.7639080291208344, 0.0681427101995416
    43
    EP 43 simulation finished, start learning
    2.3
    EP 43 learning finished: 0.5195990804360111, 0.058011248405628144
    44
    EP 44 simulation finished, start learning
    0.8
    EP 44 learning finished: 0.39023753396284633, 0.051761960776772184
    45
    EP 45 simulation finished, start learning
    -0.1
    EP 45 learning finished: 0.34719473967088765, 0.03570636729940262
    46
    EP 46 simulation finished, start learning
    1.0
    EP 46 learning finished: 0.3038569518467229, 0.031982447493999325
    47
    EP 47 simulation finished, start learning
    -0.3
    EP 47 learning finished: 0.2666412610801414, 0.032787868849717586
    48
    EP 48 simulation finished, start learning
    1.7
    EP 48 learning finished: 0.1769481435948812, 0.028799765885081027
    49
    EP 49 simulation finished, start learning
    0.6
    EP 49 learning finished: 0.16126829615424088, 0.026167996131505014
    50
    EP 50 simulation finished, start learning
    -0.3
    EP 50 learning finished: 0.2667312983862917, 0.03577634501657488
    51
    EP 51 simulation finished, start learning
    0.8
    EP 51 learning finished: 0.22043204289748708, 0.03401552847687532
    52
    EP 52 simulation finished, start learning
    0.8
    EP 52 learning finished: 0.18384828523919694, 0.032240994606782494
    53
    EP 53 simulation finished, start learning
    -0.5
    EP 53 learning finished: 0.33933531720624366, 0.04217487360815965
    54
    EP 54 simulation finished, start learning
    -0.1
    EP 54 learning finished: 0.2642047454818286, 0.030450454146084306
    55
    EP 55 simulation finished, start learning
    -0.5
    EP 55 learning finished: 0.2115962169557779, 0.026644583405443303
    56
    EP 56 simulation finished, start learning
    0.8
    EP 56 learning finished: 0.17389579792065568, 0.02635328596241447
    57
    EP 57 simulation finished, start learning
    1.0
    EP 57 learning finished: 0.12542963250996841, 0.024068527675702993
    58
    EP 58 simulation finished, start learning
    -0.5
    EP 58 learning finished: 0.21484502089198942, 0.024184347706744463
    59
    EP 59 simulation finished, start learning
    1.7
    EP 59 learning finished: 0.5138128189975983, 0.04499215795531598
    60
    EP 60 simulation finished, start learning
    0.8
    EP 60 learning finished: 0.4423022385758575, 0.0397254278820508
    61
    EP 61 simulation finished, start learning
    1.0
    EP 61 learning finished: 0.39012017320110814, 0.039569713365908935
    62
    EP 62 simulation finished, start learning
    -0.3
    EP 62 learning finished: 0.34513799033569903, 0.03946861302726833
    63
    EP 63 simulation finished, start learning
    1.0
    EP 63 learning finished: 0.2004130626356564, 0.02565518495057463
    64
    EP 64 simulation finished, start learning
    0.6
    EP 64 learning finished: 0.18970800777218266, 0.030190358789979957
    65
    EP 65 simulation finished, start learning
    -0.1
    EP 65 learning finished: 0.15495265098205713, 0.027854871766970154
    66
    EP 66 simulation finished, start learning
    0.8
    EP 66 learning finished: 0.1495766031361041, 0.02250504041725772
    67
    EP 67 simulation finished, start learning
    2.1
    EP 67 learning finished: 0.1427606264246507, 0.02080746425272389
    68
    EP 68 simulation finished, start learning
    -0.3
    EP 68 learning finished: 0.13726308857071629, 0.021410922612939445
    69
    EP 69 simulation finished, start learning
    1.0
    EP 69 learning finished: 0.12975712027256026, 0.016741972550667783
    70
    EP 70 simulation finished, start learning
    0.6
    EP 70 learning finished: 0.14871167358641335, 0.021106984700370583
    71
    EP 71 simulation finished, start learning
    0.4
    EP 71 learning finished: 0.1408471138381291, 0.028231958962878514
    72
    EP 72 simulation finished, start learning
    0.6
    EP 72 learning finished: 0.3081207671712196, 0.038082592692771935
    73
    EP 73 simulation finished, start learning
    -0.3
    EP 73 learning finished: 0.28565109710809544, 0.038123924882938046
    74
    EP 74 simulation finished, start learning
    0.4
    EP 74 learning finished: 0.4162263326128577, 0.04846985599584017
    75
    EP 75 simulation finished, start learning
    0.8
    EP 75 learning finished: 0.4814657053161113, 0.048723436922652075
    76
    EP 76 simulation finished, start learning
    -1.6
    EP 76 learning finished: 0.333160824588421, 0.04041097046477417
    77
    EP 77 simulation finished, start learning
    -0.1
    EP 77 learning finished: 0.30085396675744924, 0.040736760979707534
    78
    EP 78 simulation finished, start learning
    -0.1
    EP 78 learning finished: 0.19238039214715563, 0.0323162737469542
    79
    EP 79 simulation finished, start learning
    -0.1
    EP 79 learning finished: 0.12263861243413375, 0.025811330865271074
    80
    EP 80 simulation finished, start learning
    0.8
    EP 80 learning finished: 0.13517197415140603, 0.022564922699127084
    81
    EP 81 simulation finished, start learning
    0.6
    EP 81 learning finished: 0.12749406333493513, 0.018044707365510337
    82
    EP 82 simulation finished, start learning
    0.6
    EP 82 learning finished: 0.061606153997447066, 0.014310859877193716
    83
    EP 83 simulation finished, start learning
    0.8
    EP 83 learning finished: 0.31456689079442035, 0.02122431531546064
    84
    EP 84 simulation finished, start learning
    2.3
    EP 84 learning finished: 0.2859699007222005, 0.01705313705237027
    85
    EP 85 simulation finished, start learning
    1.7
    EP 85 learning finished: 0.2811081645675193, 0.029520016434535522
    86
    EP 86 simulation finished, start learning
    -0.5
    EP 86 learning finished: 0.2570715381379083, 0.03455416677035256
    87
    EP 87 simulation finished, start learning
    -0.3
    EP 87 learning finished: 0.23496011109629134, 0.026703907372187446
    88
    EP 88 simulation finished, start learning
    -0.1
    EP 88 learning finished: 0.21538010183826728, 0.029626848048117403
    89
    EP 89 simulation finished, start learning
    0.8
    EP 89 learning finished: 0.14714917332867716, 0.018666037843199604
    90
    EP 90 simulation finished, start learning
    -0.3
    EP 90 learning finished: 0.13816912961231342, 0.013839673893473944
    91
    EP 91 simulation finished, start learning
    1.9
    EP 91 learning finished: 0.11797069135958793, 0.015199368882263784
    92
    EP 92 simulation finished, start learning
    -0.7
    EP 92 learning finished: 0.21297901740099245, 0.02339320746045285
    93
    EP 93 simulation finished, start learning
    0.8
    EP 93 learning finished: 0.20040933555717677, 0.025197861696717547
    94
    EP 94 simulation finished, start learning
    -0.1
    EP 94 learning finished: 0.19002720045241395, 0.0276312645439541
    95
    EP 95 simulation finished, start learning
    0.1
    EP 95 learning finished: 0.1794701337606135, 0.02601286108184501
    96
    EP 96 simulation finished, start learning
    -0.3
    EP 96 learning finished: 0.16973813778913494, 0.02185034991040441
    97
    EP 97 simulation finished, start learning
    0.6
    EP 97 learning finished: 0.16182736787559415, 0.022697859742097978
    98
    EP 98 simulation finished, start learning
    0.8
    EP 98 learning finished: 0.20275512865578627, 0.02378987781292243
    99
    EP 99 simulation finished, start learning
    -0.1
    EP 99 learning finished: 0.18891836688105723, 0.027891069948804185
    100
    EP 100 simulation finished, start learning
    -0.3
    EP 100 learning finished: 0.17528682930287331, 0.02611757852273853
    101
    EP 101 simulation finished, start learning
    -0.5
    EP 101 learning finished: 0.4901134381973997, 0.03550167390531673
    102
    EP 102 simulation finished, start learning
    -0.3
    EP 102 learning finished: 0.4030007727718754, 0.029041431445011305
    103
    EP 103 simulation finished, start learning
    0.8
    EP 103 learning finished: 0.27573925559421575, 0.026201930571494128
    104
    EP 104 simulation finished, start learning
    0.6
    EP 104 learning finished: 0.24774272689931687, 0.027634280851471054
    105
    EP 105 simulation finished, start learning
    0.8
    EP 105 learning finished: 0.15809671558527683, 0.01857565570782859
    106
    EP 106 simulation finished, start learning
    0.8
    EP 106 learning finished: 0.14997929180657366, 0.0181704057260696
    107
    EP 107 simulation finished, start learning
    1.0
    EP 107 learning finished: 0.14259726297664166, 0.017148323377633314
    108
    EP 108 simulation finished, start learning
    1.0
    EP 108 learning finished: 0.06946518997607498, 0.01399913049713191
    109
    EP 109 simulation finished, start learning
    1.0
    EP 109 learning finished: 0.06841194752434987, 0.014739843028743982
    110
    EP 110 simulation finished, start learning
    0.8
    EP 110 learning finished: 0.15103210943952483, 0.02164652830005869
    111
    EP 111 simulation finished, start learning
    -0.5
    EP 111 learning finished: 0.143840104228119, 0.023327555825183036
    112
    EP 112 simulation finished, start learning
    0.8
    EP 112 learning finished: 0.13816409088283255, 0.021935941420378996
    113
    EP 113 simulation finished, start learning
    -0.3
    EP 113 learning finished: 0.1367285583824132, 0.025569969650047444
    114
    EP 114 simulation finished, start learning
    1.0
    EP 114 learning finished: 0.25660483243270543, 0.026963448418569073
    115
    EP 115 simulation finished, start learning
    -0.5
    EP 115 learning finished: 0.2868339450116275, 0.031592064991927515
    116
    EP 116 simulation finished, start learning
    0.8
    EP 116 learning finished: 0.2629311162606587, 0.028790546476907927
    117
    EP 117 simulation finished, start learning
    -0.3
    EP 117 learning finished: 0.2420769399738053, 0.027821853715360122
    118
    EP 118 simulation finished, start learning
    1.0
    EP 118 learning finished: 0.22397197535854774, 0.02136270940405573
    119
    EP 119 simulation finished, start learning
    -0.5
    EP 119 learning finished: 0.12028867027022949, 0.015316275202255508
    120
    EP 120 simulation finished, start learning
    0.4
    EP 120 learning finished: 0.11629056922537928, 0.019751616840612753
    121
    EP 121 simulation finished, start learning
    0.8
    EP 121 learning finished: 0.1255451825490681, 0.018363053306597626
    122
    EP 122 simulation finished, start learning
    0.8
    EP 122 learning finished: 0.12565940256338592, 0.018836791735837535
    123
    EP 123 simulation finished, start learning
    0.8
    EP 123 learning finished: 0.1185326498788477, 0.016512899581869356
    124
    EP 124 simulation finished, start learning
    -0.1
    EP 124 learning finished: 0.11192073647692702, 0.010661422455615133
    125
    EP 125 simulation finished, start learning
    0.8
    EP 125 learning finished: 0.10541032751943202, 0.009787086490079986
    126
    EP 126 simulation finished, start learning
    -0.3
    EP 126 learning finished: 0.05551374077863713, 0.007180734458461933
    127
    EP 127 simulation finished, start learning
    0.1
    EP 127 learning finished: 0.10283548677936993, 0.010014691371114187
    128
    EP 128 simulation finished, start learning
    0.8
    EP 128 learning finished: 0.09999988082337707, 0.011813778857756656
    129
    EP 129 simulation finished, start learning
    0.4
    EP 129 learning finished: 0.10098673221839238, 0.012632533918875657
    130
    EP 130 simulation finished, start learning
    -0.5
    EP 130 learning finished: 0.12115030676296268, 0.01648963824330711
    131
    EP 131 simulation finished, start learning
    -0.3
    EP 131 learning finished: 0.11792172469868478, 0.016477835152952224
    132
    EP 132 simulation finished, start learning
    0.8
    EP 132 learning finished: 0.11377982177919899, 0.016178280470445457
    133
    EP 133 simulation finished, start learning
    -0.5
    EP 133 learning finished: 0.10950309371534028, 0.017122596750240447
    134
    EP 134 simulation finished, start learning
    1.2
    EP 134 learning finished: 0.10635617956079924, 0.0125112156344095
    135
    EP 135 simulation finished, start learning
    0.8
    EP 135 learning finished: 0.06098337844750934, 0.012313543106534382
    136
    EP 136 simulation finished, start learning
    1.0
    EP 136 learning finished: 0.09401294207001021, 0.014453877992033171
    137
    EP 137 simulation finished, start learning
    -0.5
    EP 137 learning finished: 0.0905309812526025, 0.013208265062999068
    138
    EP 138 simulation finished, start learning
    -0.5
    EP 138 learning finished: 0.08785124408458711, 0.016602557252621934
    139
    EP 139 simulation finished, start learning
    0.4
    EP 139 learning finished: 0.15578083298081502, 0.015065998844147499
    140
    EP 140 simulation finished, start learning
    0.4
    EP 140 learning finished: 0.14211404877548617, 0.01224999504497773
    141
    EP 141 simulation finished, start learning
    -0.1
    EP 141 learning finished: 0.12995093077021913, 0.010812628230523924
    142
    EP 142 simulation finished, start learning
    0.8
    EP 142 learning finished: 0.06093807681595642, 0.009359762505735246
    143
    EP 143 simulation finished, start learning
    0.4
    EP 143 learning finished: 0.04072148652955376, 0.008520248600686217
    144
    EP 144 simulation finished, start learning
    0.8
    EP 144 learning finished: 0.05838877010800303, 0.0093562187119349
    145
    EP 145 simulation finished, start learning
    0.8
    EP 145 learning finished: 0.056337539760315636, 0.009809227594296544
    146
    EP 146 simulation finished, start learning
    -0.3
    EP 146 learning finished: 0.05412822447559762, 0.010126127979357732
    147
    EP 147 simulation finished, start learning
    -0.3
    EP 147 learning finished: 0.05706787141698355, 0.010525107244938842
    148
    EP 148 simulation finished, start learning
    1.0
    EP 148 learning finished: 0.056028030947149365, 0.009952507608993511
    149
    EP 149 simulation finished, start learning
    0.8
    EP 149 learning finished: 0.07432642204200568, 0.009996247865489742
    150
    EP 150 simulation finished, start learning
    -0.3
    EP 150 learning finished: 0.07126149249354419, 0.01049091805572109
    151
    EP 151 simulation finished, start learning
    0.8
    EP 151 learning finished: 0.0685494488500531, 0.010383805778240947
    152
    EP 152 simulation finished, start learning
    1.9
    EP 152 learning finished: 0.0518918743976291, 0.010723368764479067
    153
    EP 153 simulation finished, start learning
    -0.1
    EP 153 learning finished: 0.051112104374128164, 0.010474302880058376
    154
    EP 154 simulation finished, start learning
    0.8
    EP 154 learning finished: 0.05012917929001026, 0.009594821609544134
    155
    EP 155 simulation finished, start learning
    -0.7
    EP 155 learning finished: 0.045804560077743084, 0.008858416240321772
    156
    EP 156 simulation finished, start learning
    2.1
    EP 156 learning finished: 0.04492982423610259, 0.007657327861762004
    157
    EP 157 simulation finished, start learning
    0.6
    EP 157 learning finished: 0.037930208082666894, 0.007778892926549541
    158
    EP 158 simulation finished, start learning
    -0.3
    EP 158 learning finished: 0.05575707706188626, 0.006858399692391
    159
    EP 159 simulation finished, start learning
    -0.5
    EP 159 learning finished: 0.1272948970338108, 0.011598266973726325
    160
    EP 160 simulation finished, start learning
    0.8
    EP 160 learning finished: 0.12078213965135465, 0.014786224128665391
    161
    EP 161 simulation finished, start learning
    1.9
    EP 161 learning finished: 0.27591168828550394, 0.023469420244773116
    162
    EP 162 simulation finished, start learning
    0.6
    EP 162 learning finished: 0.26567096761893794, 0.023904075447735845
    163
    EP 163 simulation finished, start learning
    3.2
    EP 163 learning finished: 0.25635389892067906, 0.022530331804766504
    164
    EP 164 simulation finished, start learning
    -0.5
    EP 164 learning finished: 0.24761813733001148, 0.02053745844001784
    165
    EP 165 simulation finished, start learning
    0.8
    EP 165 learning finished: 0.09644441417911764, 0.01207352967494562
    166
    EP 166 simulation finished, start learning
    0.1
    EP 166 learning finished: 0.09439036792391953, 0.010624060851940374
    167
    EP 167 simulation finished, start learning
    -0.5
    EP 167 learning finished: 0.04961802281690375, 0.00820393317592628
    168
    EP 168 simulation finished, start learning
    0.8
    EP 168 learning finished: 0.09713881708280536, 0.008413960568109065
    169
    EP 169 simulation finished, start learning
    0.4
    EP 169 learning finished: 0.09396944651908096, 0.010864448902738777
    170
    EP 170 simulation finished, start learning
    2.1
    EP 170 learning finished: 0.09090256203709213, 0.010378719914701597
    171
    EP 171 simulation finished, start learning
    -0.1
    EP 171 learning finished: 0.21594987910082675, 0.018152184648338184
    172
    EP 172 simulation finished, start learning
    -0.5
    EP 172 learning finished: 0.2006440038672094, 0.02010072926353697
    173
    EP 173 simulation finished, start learning
    1.7
    EP 173 learning finished: 0.18692116158797564, 0.018521781963886475
    174
    EP 174 simulation finished, start learning
    -0.5
    EP 174 learning finished: 0.17468077283339245, 0.019022276628105638
    175
    EP 175 simulation finished, start learning
    -0.5
    EP 175 learning finished: 0.09706974496611287, 0.011352555692743026
    176
    EP 176 simulation finished, start learning
    -0.3
    EP 176 learning finished: 0.07345154653150188, 0.00884065048162053
    177
    EP 177 simulation finished, start learning
    -0.1
    EP 177 learning finished: 0.07081144807969075, 0.008684323506340245
    178
    EP 178 simulation finished, start learning
    -0.3
    EP 178 learning finished: 0.07181735566679848, 0.008987378045862044
    179
    EP 179 simulation finished, start learning
    -0.5
    EP 179 learning finished: 0.045636561610924886, 0.00960927504906446
    180
    EP 180 simulation finished, start learning
    0.8
    EP 180 learning finished: 0.04404936158788786, 0.009976237960881675
    181
    EP 181 simulation finished, start learning
    -0.1
    EP 181 learning finished: 0.09478348721078633, 0.010864277525410658
    182
    EP 182 simulation finished, start learning
    -0.1
    EP 182 learning finished: 0.09172726853380819, 0.010151075160790038
    183
    EP 183 simulation finished, start learning
    0.8
    EP 183 learning finished: 0.08892770455712457, 0.009371150695096938
    184
    EP 184 simulation finished, start learning
    1.0
    EP 184 learning finished: 0.08606407217422041, 0.007835260282074683
    185
    EP 185 simulation finished, start learning
    3.0
    EP 185 learning finished: 0.2099674057812564, 0.012213151218081988
    186
    EP 186 simulation finished, start learning
    -0.5
    EP 186 learning finished: 0.20354198529527423, 0.016269914134843695
    187
    EP 187 simulation finished, start learning
    0.8
    EP 187 learning finished: 0.19727912643418577, 0.017035667907813856
    188
    EP 188 simulation finished, start learning
    -0.3
    EP 188 learning finished: 0.1916528797211683, 0.019827780214212845
    189
    EP 189 simulation finished, start learning
    1.0
    EP 189 learning finished: 0.13840586297442992, 0.014989190775875173
    190
    EP 190 simulation finished, start learning
    0.6
    EP 190 learning finished: 0.07466658789685932, 0.01087937283833397
    191
    EP 191 simulation finished, start learning
    -0.3
    EP 191 learning finished: 0.08439582783835026, 0.013331343892285649
    192
    EP 192 simulation finished, start learning
    0.8
    EP 192 learning finished: 0.08194893398098274, 0.01027085647332371
    193
    EP 193 simulation finished, start learning
    -0.3
    EP 193 learning finished: 0.07952575407954708, 0.008902102407678525
    194
    EP 194 simulation finished, start learning
    1.2
    EP 194 learning finished: 0.07712789211121948, 0.012732305336223416
    195
    EP 195 simulation finished, start learning
    -0.7
    EP 195 learning finished: 0.131497817172332, 0.012948406610315238
    196
    EP 196 simulation finished, start learning
    0.6
    EP 196 learning finished: 0.12435181886940194, 0.014261048691419365
    197
    EP 197 simulation finished, start learning
    -1.6
    EP 197 learning finished: 0.1314936972444939, 0.01710685880404809
    198
    EP 198 simulation finished, start learning
    -0.3
    EP 198 learning finished: 0.12422457392581465, 0.014611151010270839
    199
    EP 199 simulation finished, start learning
    1.0
    EP 199 learning finished: 0.1175097320919869, 0.011142444281364204
    200
    EP 200 simulation finished, start learning
    0.8
    EP 200 learning finished: 0.11108146499807536, 0.009622813213546029
    201
    EP 201 simulation finished, start learning
    -0.3
    EP 201 learning finished: 0.06709119647803163, 0.007363955753461812
    202
    EP 202 simulation finished, start learning
    0.8
    EP 202 learning finished: 0.05110168166010265, 0.006568048007612029
    203
    EP 203 simulation finished, start learning
    -0.5
    EP 203 learning finished: 0.0313695426218743, 0.006212604937198801
    204
    EP 204 simulation finished, start learning
    -0.3
    EP 204 learning finished: 0.049637165646441606, 0.0061817776608882324
    205
    EP 205 simulation finished, start learning
    -0.1
    EP 205 learning finished: 0.04807622714517423, 0.005831156595431125
    206
    EP 206 simulation finished, start learning
    1.0
    EP 206 learning finished: 0.058144733736292586, 0.007798518014768344
    207
    EP 207 simulation finished, start learning
    -0.5
    EP 207 learning finished: 0.08422981930567985, 0.008770895138924081
    208
    EP 208 simulation finished, start learning
    -0.1
    EP 208 learning finished: 0.10446818868099195, 0.011033738069037215
    209
    EP 209 simulation finished, start learning
    -0.5
    EP 209 learning finished: 0.0993086852080114, 0.011542647603139791
    210
    EP 210 simulation finished, start learning
    0.8
    EP 210 learning finished: 0.05035974440119784, 0.010094986562986515
    211
    EP 211 simulation finished, start learning
    -0.5
    EP 211 learning finished: 0.06187682977609121, 0.013548917498335453
    212
    EP 212 simulation finished, start learning
    1.0
    EP 212 learning finished: 0.1529671292962984, 0.015284425712263058
    213
    EP 213 simulation finished, start learning
    -0.1
    EP 213 learning finished: 0.14606286185136952, 0.013429243740052721
    214
    EP 214 simulation finished, start learning
    1.9
    EP 214 learning finished: 0.13981919138882404, 0.012327587872695625
    215
    EP 215 simulation finished, start learning
    -0.3
    EP 215 learning finished: 0.1339253419166404, 0.008879735710108874
    216
    EP 216 simulation finished, start learning
    0.8
    EP 216 learning finished: 0.038358977228925895, 0.005427816193882336
    217
    EP 217 simulation finished, start learning
    0.8
    EP 217 learning finished: 0.1285573664184767, 0.009599496064005404
    218
    EP 218 simulation finished, start learning
    0.8
    EP 218 learning finished: 0.1236197381087436, 0.012155785107716205
    219
    EP 219 simulation finished, start learning
    0.8
    EP 219 learning finished: 0.11869562673749279, 0.010508033887397975
    220
    EP 220 simulation finished, start learning
    0.8
    EP 220 learning finished: 0.11416687856084629, 0.01056461897733338
    221
    EP 221 simulation finished, start learning
    1.0
    EP 221 learning finished: 0.07196173187145005, 0.006359156635762295
    222
    EP 222 simulation finished, start learning
    -0.1
    EP 222 learning finished: 0.027493191109462245, 0.004396713421422227
    223
    EP 223 simulation finished, start learning
    -0.3
    EP 223 learning finished: 0.041834561501529116, 0.00599309709668532
    224
    EP 224 simulation finished, start learning
    0.6
    EP 224 learning finished: 0.0411861592196337, 0.007944923156545687
    225
    EP 225 simulation finished, start learning
    -0.3
    EP 225 learning finished: 0.054083178026314016, 0.010283089649097417
    226
    EP 226 simulation finished, start learning
    -0.1
    EP 226 learning finished: 0.05237331572595849, 0.010867925197965997
    227
    EP 227 simulation finished, start learning
    0.6
    EP 227 learning finished: 0.05049544339535661, 0.00972651270920203
    228
    EP 228 simulation finished, start learning
    1.0
    EP 228 learning finished: 0.04863915717011302, 0.007488841142778348
    229
    EP 229 simulation finished, start learning
    0.6
    EP 229 learning finished: 0.044433059976228595, 0.006484694856097581
    230
    EP 230 simulation finished, start learning
    0.8
    EP 230 learning finished: 0.02610929852649413, 0.005397261273493395
    231
    EP 231 simulation finished, start learning
    0.8
    EP 231 learning finished: 0.02552229859936972, 0.005232999540261497
    232
    EP 232 simulation finished, start learning
    1.0
    EP 232 learning finished: 0.07382191692376328, 0.00789833925730938
    233
    EP 233 simulation finished, start learning
    1.0
    EP 233 learning finished: 0.07108107514218442, 0.0067254774630678
    234
    EP 234 simulation finished, start learning
    1.0
    EP 234 learning finished: 0.06844823390970678, 0.0063999103547051905
    235
    EP 235 simulation finished, start learning
    -0.5
    EP 235 learning finished: 0.06613002514292354, 0.007064135660943121
    236
    EP 236 simulation finished, start learning
    0.8
    EP 236 learning finished: 0.03952574469257786, 0.004692250742049368
    237
    EP 237 simulation finished, start learning
    -0.3
    EP 237 learning finished: 0.03893564148856843, 0.006061376302842425
    238
    EP 238 simulation finished, start learning
    0.8
    EP 238 learning finished: 0.0514402578883244, 0.0065321010802024335
    239
    EP 239 simulation finished, start learning
    2.3
    EP 239 learning finished: 0.04994054922582336, 0.005373660743260886
    240
    EP 240 simulation finished, start learning
    -0.1
    EP 240 learning finished: 0.0881589093124413, 0.007765808258658538
    241
    EP 241 simulation finished, start learning
    -0.5
    EP 241 learning finished: 0.08438097085058427, 0.007477822983838103
    242
    EP 242 simulation finished, start learning
    0.4
    EP 242 learning finished: 0.08071991437657156, 0.0070875611376173125
    243
    EP 243 simulation finished, start learning
    0.8
    EP 243 learning finished: 0.07759853148378326, 0.009109346233245448
    244
    EP 244 simulation finished, start learning
    0.8
    EP 244 learning finished: 0.07063233615431208, 0.006507500604293389
    245
    EP 245 simulation finished, start learning
    -0.3
    EP 245 learning finished: 0.06886264632931915, 0.0058749614213668006
    246
    EP 246 simulation finished, start learning
    -0.9
    EP 246 learning finished: 0.06705716436502085, 0.008240545649445277
    247
    EP 247 simulation finished, start learning
    1.0
    EP 247 learning finished: 0.05040973620952105, 0.00672018722933769
    248
    EP 248 simulation finished, start learning
    2.1
    EP 248 learning finished: 0.04910144165068786, 0.006348007229006539
    249
    EP 249 simulation finished, start learning
    -0.7
    EP 249 learning finished: 0.04765728160213811, 0.006264581827651772
    250
    EP 250 simulation finished, start learning
    -0.3
    EP 250 learning finished: 0.046365156358315396, 0.005336906606886516
    251
    EP 251 simulation finished, start learning
    1.0
    EP 251 learning finished: 0.04516321777946164, 0.006069606432814678
    252
    EP 252 simulation finished, start learning
    -0.5
    EP 252 learning finished: 0.043910697920730435, 0.007000142544040619
    253
    EP 253 simulation finished, start learning
    0.6
    EP 253 learning finished: 0.04265680407414596, 0.007016165091309423
    254
    EP 254 simulation finished, start learning
    -0.1
    EP 254 learning finished: 0.0355915314248576, 0.006542745897417435
    255
    EP 255 simulation finished, start learning
    -0.3
    EP 255 learning finished: 0.037517058517772206, 0.005455341723084456
    256
    EP 256 simulation finished, start learning
    -0.1
    EP 256 learning finished: 0.033574931870227864, 0.005768177448888606
    257
    EP 257 simulation finished, start learning
    0.8
    EP 257 learning finished: 0.06570328031204653, 0.009076898123677013
    258
    EP 258 simulation finished, start learning
    0.4
    EP 258 learning finished: 0.06379800556495274, 0.008820742311759916
    259
    EP 259 simulation finished, start learning
    -0.5
    EP 259 learning finished: 0.15604217814166876, 0.011523273462294939
    260
    EP 260 simulation finished, start learning
    -0.1
    EP 260 learning finished: 0.17242207896880135, 0.01140298219372916
    261
    EP 261 simulation finished, start learning
    1.2
    EP 261 learning finished: 0.10666335875368649, 0.007801832540287226
    262
    EP 262 simulation finished, start learning
    0.6
    EP 262 learning finished: 0.10023778478461232, 0.010330493801521276
    263
    EP 263 simulation finished, start learning
    1.9
    EP 263 learning finished: 0.09725333895968191, 0.007898156079398658
    264
    EP 264 simulation finished, start learning
    0.6
    EP 264 learning finished: 0.09416306458096102, 0.007786041922400622
    265
    EP 265 simulation finished, start learning
    -0.3
    EP 265 learning finished: 0.09152666116428776, 0.00887572550178578
    266
    EP 266 simulation finished, start learning
    -0.3
    EP 266 learning finished: 0.047288673333894415, 0.008213689133630749
    267
    EP 267 simulation finished, start learning
    2.3
    EP 267 learning finished: 0.03854172114414567, 0.0078104349161375874
    268
    EP 268 simulation finished, start learning
    0.8
    EP 268 learning finished: 0.038007702052517356, 0.006785514147210278
    269
    EP 269 simulation finished, start learning
    0.6
    EP 269 learning finished: 0.11287636894644826, 0.011338934237505535
    270
    EP 270 simulation finished, start learning
    1.0
    EP 270 learning finished: 0.10734410737113542, 0.008901968825593812
    271
    EP 271 simulation finished, start learning
    -0.3
    EP 271 learning finished: 0.10221587410612476, 0.009144043058861483
    272
    EP 272 simulation finished, start learning
    -0.3
    EP 272 learning finished: 0.09760931997216882, 0.009509384842665163
    273
    EP 273 simulation finished, start learning
    1.9
    EP 273 learning finished: 0.03384217720755034, 0.0040504554976908
    274
    EP 274 simulation finished, start learning
    -0.3
    EP 274 learning finished: 0.033100567246164836, 0.005536391525610751
    275
    EP 275 simulation finished, start learning
    0.8
    EP 275 learning finished: 0.030789671559865894, 0.005143789008900501
    276
    EP 276 simulation finished, start learning
    0.8
    EP 276 learning finished: 0.0738680229081119, 0.007467520264652278
    277
    EP 277 simulation finished, start learning
    -0.1
    EP 277 learning finished: 0.07225445901163319, 0.007641320736256188
    278
    EP 278 simulation finished, start learning
    0.8
    EP 278 learning finished: 0.07060970644592679, 0.0064810522662898384
    279
    EP 279 simulation finished, start learning
    -0.3
    EP 279 learning finished: 0.06908004798434852, 0.008512057960924444
    280
    EP 280 simulation finished, start learning
    0.6
    EP 280 learning finished: 0.06036059897468071, 0.007883344946747386
    281
    EP 281 simulation finished, start learning
    1.9
    EP 281 learning finished: 0.06841414733543516, 0.009858940842539854
    282
    EP 282 simulation finished, start learning
    1.0
    EP 282 learning finished: 0.0670510557858286, 0.009505670112563404
    283
    EP 283 simulation finished, start learning
    0.6
    EP 283 learning finished: 0.06579179321053985, 0.007398687014772268
    284
    EP 284 simulation finished, start learning
    1.2
    EP 284 learning finished: 0.06452386073870287, 0.00688538338004463
    285
    EP 285 simulation finished, start learning
    3.0
    EP 285 learning finished: 0.08293662838829974, 0.00596562312278516
    286
    EP 286 simulation finished, start learning
    0.8
    EP 286 learning finished: 0.08111800636068622, 0.006693050274036272
    287
    EP 287 simulation finished, start learning
    -0.3
    EP 287 learning finished: 0.07940787500863422, 0.007621270278062614
    288
    EP 288 simulation finished, start learning
    -0.5
    EP 288 learning finished: 0.05688915839563791, 0.006740576810239943
    289
    EP 289 simulation finished, start learning
    -0.1
    EP 289 learning finished: 0.03759551469313127, 0.006310885112358711
    290
    EP 290 simulation finished, start learning
    -0.1
    EP 290 learning finished: 0.037093933043430005, 0.005563995219840603
    291
    EP 291 simulation finished, start learning
    0.8
    EP 291 learning finished: 0.03656842391939186, 0.004819374270026541
    292
    EP 292 simulation finished, start learning
    0.8
    EP 292 learning finished: 0.03605446652314792, 0.004550051749896526
    293
    EP 293 simulation finished, start learning
    0.6
    EP 293 learning finished: 0.049326425143155106, 0.0053841953026451395
    294
    EP 294 simulation finished, start learning
    1.0
    EP 294 learning finished: 0.0626980875001717, 0.006962693304188583
    295
    EP 295 simulation finished, start learning
    0.8
    EP 295 learning finished: 0.06379188732289975, 0.006931699630466068
    296
    EP 296 simulation finished, start learning
    2.1
    EP 296 learning finished: 0.06253082552988509, 0.007210851968906626
    297
    EP 297 simulation finished, start learning
    -0.1
    EP 297 learning finished: 0.061342958413735715, 0.006485965898216123
    298
    EP 298 simulation finished, start learning
    0.8
    EP 298 learning finished: 0.031325846092101095, 0.004853668380418452
    299
    EP 299 simulation finished, start learning
    1.0
    EP 299 learning finished: 0.03079903559752495, 0.004531176209748661
    300
    EP 300 simulation finished, start learning
    1.7
    EP 300 learning finished: 0.031723640451582824, 0.004420633333279217
    301
    EP 301 simulation finished, start learning
    3.2
    EP 301 learning finished: 0.0529696343272974, 0.0049153706082070425
    302
    EP 302 simulation finished, start learning
    -0.5
    EP 302 learning finished: 0.052571390862247824, 0.004920008026365816
    303
    EP 303 simulation finished, start learning
    -0.1
    EP 303 learning finished: 0.05222553621549508, 0.005010633330944106
    304
    EP 304 simulation finished, start learning
    1.0
    EP 304 learning finished: 0.05340729248144527, 0.005593243743007709
    305
    EP 305 simulation finished, start learning
    -0.1
    EP 305 learning finished: 0.029522725304746977, 0.004115417439378377
    306
    EP 306 simulation finished, start learning
    -0.1
    EP 306 learning finished: 0.029072526233000495, 0.004233642052952236
    307
    EP 307 simulation finished, start learning
    1.9
    EP 307 learning finished: 0.028580242209153006, 0.004173032241578001
    308
    EP 308 simulation finished, start learning
    0.6
    EP 308 learning finished: 0.041264472753329784, 0.003993081620247002
    309
    EP 309 simulation finished, start learning
    -0.3
    EP 309 learning finished: 0.03863915154486719, 0.006096463954588929
    310
    EP 310 simulation finished, start learning
    0.8
    EP 310 learning finished: 0.036834855342199546, 0.005758712104673144
    311
    EP 311 simulation finished, start learning
    0.8
    EP 311 learning finished: 0.035987685133089364, 0.006205595535235153
    312
    EP 312 simulation finished, start learning
    -0.1
    EP 312 learning finished: 0.03526959187695322, 0.005832951627747347
    313
    EP 313 simulation finished, start learning
    -0.1
    EP 313 learning finished: 0.03317121376381227, 0.0038609248826715625
    314
    EP 314 simulation finished, start learning
    0.6
    EP 314 learning finished: 0.04106906475455041, 0.004790893146753142
    315
    EP 315 simulation finished, start learning
    -0.1
    EP 315 learning finished: 0.040105071025829275, 0.005254829234380587
    316
    EP 316 simulation finished, start learning
    0.4
    EP 316 learning finished: 0.039133648270903265, 0.005137171159613874
    317
    EP 317 simulation finished, start learning
    0.6
    EP 317 learning finished: 0.038176977789254884, 0.0059209488191561965
    318
    EP 318 simulation finished, start learning
    0.6
    EP 318 learning finished: 0.02944740303270077, 0.00482889429399086
    319
    EP 319 simulation finished, start learning
    0.8
    EP 319 learning finished: 0.034909281057272756, 0.00494176874485967
    320
    EP 320 simulation finished, start learning
    -0.5
    EP 320 learning finished: 0.03441760104238156, 0.005128834110660424
    321
    EP 321 simulation finished, start learning
    0.6
    EP 321 learning finished: 0.03475304738148477, 0.00531238233554257
    322
    EP 322 simulation finished, start learning
    0.8
    EP 322 learning finished: 0.0340184873096856, 0.005422343361542362
    323
    EP 323 simulation finished, start learning
    0.8
    EP 323 learning finished: 0.06586027628427082, 0.005291276407284551
    324
    EP 324 simulation finished, start learning
    -0.5
    EP 324 learning finished: 0.06300265092796486, 0.006129812124520809
    325
    EP 325 simulation finished, start learning
    0.1
    EP 325 learning finished: 0.03056449177820575, 0.0056362215875735006
    326
    EP 326 simulation finished, start learning
    1.9
    EP 326 learning finished: 0.029966390251304986, 0.0054987628277954476
    327
    EP 327 simulation finished, start learning
    0.4
    EP 327 learning finished: 0.02642492451407108, 0.004769200614712879
    328
    EP 328 simulation finished, start learning
    2.1
    EP 328 learning finished: 0.04827900289447129, 0.0047547355486747545
    329
    EP 329 simulation finished, start learning
    0.8
    EP 329 learning finished: 0.04939097932127012, 0.0046063242697468865
    330
    EP 330 simulation finished, start learning
    0.8
    EP 330 learning finished: 0.05002724956569082, 0.004908914496640646
    331
    EP 331 simulation finished, start learning
    0.4
    EP 331 learning finished: 0.049108682883098, 0.005072630640697101
    332
    EP 332 simulation finished, start learning
    0.8
    EP 332 learning finished: 0.0573133490670763, 0.005647082981541748
    333
    EP 333 simulation finished, start learning
    1.9
    EP 333 learning finished: 0.05640985147510236, 0.005213157997540047
    334
    EP 334 simulation finished, start learning
    -0.3
    EP 334 learning finished: 0.05555962308290319, 0.005303245859035341
    335
    EP 335 simulation finished, start learning
    0.8
    EP 335 learning finished: 0.10838480480739321, 0.006738762667985038
    336
    EP 336 simulation finished, start learning
    1.0
    EP 336 learning finished: 0.05280697668713508, 0.005305608922446248
    337
    EP 337 simulation finished, start learning
    -0.3
    EP 337 learning finished: 0.05191229732774971, 0.005233765522583561
    338
    EP 338 simulation finished, start learning
    0.8
    EP 338 learning finished: 0.05101595507948531, 0.004960885644203217
    339
    EP 339 simulation finished, start learning
    0.6
    EP 339 learning finished: 0.058139421711139505, 0.004310697205906322
    340
    EP 340 simulation finished, start learning
    1.2
    EP 340 learning finished: 0.05477918861697084, 0.005918501946862581
    341
    EP 341 simulation finished, start learning
    -0.3
    EP 341 learning finished: 0.06409026347698443, 0.006668437223664281
    342
    EP 342 simulation finished, start learning
    0.6
    EP 342 learning finished: 0.04814120373905695, 0.006536764826578445
    343
    EP 343 simulation finished, start learning
    -0.3
    EP 343 learning finished: 0.04719749584672428, 0.006034463689475217
    344
    EP 344 simulation finished, start learning
    -0.1
    EP 344 learning finished: 0.04338343023908564, 0.00456580634298516
    345
    EP 345 simulation finished, start learning
    0.8
    EP 345 learning finished: 0.021225096561257928, 0.0038090805902859515
    346
    EP 346 simulation finished, start learning
    -0.3
    EP 346 learning finished: 0.025667559569593656, 0.0037988862957067937
    347
    EP 347 simulation finished, start learning
    -0.1
    EP 347 learning finished: 0.029001013779372364, 0.0038243270814002778
    348
    EP 348 simulation finished, start learning
    1.9
    EP 348 learning finished: 0.028333185806894257, 0.003191794194777761
    349
    EP 349 simulation finished, start learning
    0.8
    EP 349 learning finished: 0.023487095445371864, 0.003313860939155297
    350
    EP 350 simulation finished, start learning
    1.2
    EP 350 learning finished: 0.022920044348264668, 0.0028451320456822637
    351
    EP 351 simulation finished, start learning
    1.0
    EP 351 learning finished: 0.022342450911322587, 0.002461996439235531
    352
    EP 352 simulation finished, start learning
    1.9
    EP 352 learning finished: 0.02988016161521312, 0.0031517008095928445
    353
    EP 353 simulation finished, start learning
    -0.3
    EP 353 learning finished: 0.029260313328183685, 0.0029807993821060474
    354
    EP 354 simulation finished, start learning
    1.7
    EP 354 learning finished: 0.028660054577762217, 0.0033505733543212207
    355
    EP 355 simulation finished, start learning
    -1.4
    EP 355 learning finished: 0.028078527528268893, 0.0035803742026396523
    356
    EP 356 simulation finished, start learning
    1.0
    EP 356 learning finished: 0.023760822416878824, 0.0028314559299002506
    357
    EP 357 simulation finished, start learning
    -0.1
    EP 357 learning finished: 0.0165188504842555, 0.002856921646947383
    358
    EP 358 simulation finished, start learning
    1.7
    EP 358 learning finished: 0.01634855305658256, 0.0026381743206935343
    359
    EP 359 simulation finished, start learning
    -0.3
    EP 359 learning finished: 0.032320690418623954, 0.0036938431465134147
    360
    EP 360 simulation finished, start learning
    0.6
    EP 360 learning finished: 0.09784932744628216, 0.008425939128932247
    361
    EP 361 simulation finished, start learning
    -0.1
    EP 361 learning finished: 0.09361410867793496, 0.007913776635194054
    362
    EP 362 simulation finished, start learning
    1.0
    EP 362 learning finished: 0.08949476585951355, 0.007770342404892182
    363
    EP 363 simulation finished, start learning
    -0.7
    EP 363 learning finished: 0.08584345824814865, 0.009364422230456353
    364
    EP 364 simulation finished, start learning
    1.0
    EP 364 learning finished: 0.058574107183053226, 0.005035069719203283
    365
    EP 365 simulation finished, start learning
    -0.3
    EP 365 learning finished: 0.057220109661057394, 0.005530322347731986
    366
    EP 366 simulation finished, start learning
    1.0
    EP 366 learning finished: 0.055914351483058455, 0.0056208997260376314
    367
    EP 367 simulation finished, start learning
    2.1
    EP 367 learning finished: 0.024423297808958466, 0.0036164018253644487
    368
    EP 368 simulation finished, start learning
    0.8
    EP 368 learning finished: 0.02429293357969442, 0.003633601972025169
    369
    EP 369 simulation finished, start learning
    -0.3
    EP 369 learning finished: 0.025248148586094032, 0.003089609144763819
    370
    EP 370 simulation finished, start learning
    1.0
    EP 370 learning finished: 0.025975153451085475, 0.0035866275159275735
    371
    EP 371 simulation finished, start learning
    -0.1
    EP 371 learning finished: 0.05479254828314106, 0.004346138887894216
    372
    EP 372 simulation finished, start learning
    1.0
    EP 372 learning finished: 0.053599582121215406, 0.0048642842121650876
    373
    EP 373 simulation finished, start learning
    -0.1
    EP 373 learning finished: 0.052579228748860896, 0.005148957529669887
    374
    EP 374 simulation finished, start learning
    1.0
    EP 374 learning finished: 0.051410165182894874, 0.004706929944033496
    375
    EP 375 simulation finished, start learning
    -0.1
    EP 375 learning finished: 0.029541413658382698, 0.003335405146551826
    376
    EP 376 simulation finished, start learning
    -0.3
    EP 376 learning finished: 0.02901048925336358, 0.003157309273039005
    377
    EP 377 simulation finished, start learning
    -0.1
    EP 377 learning finished: 0.01432518572360264, 0.003006826977527466
    378
    EP 378 simulation finished, start learning
    0.8
    EP 378 learning finished: 0.014221011671253336, 0.003309075316703061
    379
    EP 379 simulation finished, start learning
    -0.5
    EP 379 learning finished: 0.0330156920813085, 0.00442084817618249
    380
    EP 380 simulation finished, start learning
    0.8
    EP 380 learning finished: 0.032589953488489254, 0.004240710259278455
    381
    EP 381 simulation finished, start learning
    0.6
    EP 381 learning finished: 0.03220125759301973, 0.003909895756636979
    382
    EP 382 simulation finished, start learning
    0.8
    EP 382 learning finished: 0.031791727955257976, 0.0036947346420668353
    383
    EP 383 simulation finished, start learning
    -0.5
    EP 383 learning finished: 0.0325349787353173, 0.0027643146955276393
    384
    EP 384 simulation finished, start learning
    -0.1
    EP 384 learning finished: 0.03170624185625126, 0.0030597361129344747
    385
    EP 385 simulation finished, start learning
    0.8
    EP 385 learning finished: 0.037981373388156214, 0.004467352892666539
    386
    EP 386 simulation finished, start learning
    -0.1
    EP 386 learning finished: 0.03700471372213965, 0.00432549802759246
    387
    EP 387 simulation finished, start learning
    0.8
    EP 387 learning finished: 0.04808787806923931, 0.005365704642979946
    388
    EP 388 simulation finished, start learning
    -0.5
    EP 388 learning finished: 0.08391809990390486, 0.0077241350727309185
    389
    EP 389 simulation finished, start learning
    0.8
    EP 389 learning finished: 0.08052633321053415, 0.006390982975998133
    390
    EP 390 simulation finished, start learning
    -0.3
    EP 390 learning finished: 0.0774096356629359, 0.00651605872856853
    391
    EP 391 simulation finished, start learning
    -0.5
    EP 391 learning finished: 0.1460356510785279, 0.008207033988810834
    392
    EP 392 simulation finished, start learning
    0.8
    EP 392 learning finished: 0.06886840309049713, 0.006040177397963045
    393
    EP 393 simulation finished, start learning
    -0.1
    EP 393 learning finished: 0.06626959542670496, 0.00626300475408046
    394
    EP 394 simulation finished, start learning
    1.9
    EP 394 learning finished: 0.06393632031796148, 0.005636606998933473
    395
    EP 395 simulation finished, start learning
    -0.3
    EP 395 learning finished: 0.020335001215773474, 0.003090293270288688
    396
    EP 396 simulation finished, start learning
    1.0
    EP 396 learning finished: 0.027019931349550497, 0.0034466688174816518
    397
    EP 397 simulation finished, start learning
    1.9
    EP 397 learning finished: 0.03310234141372703, 0.004089627319230876
    398
    EP 398 simulation finished, start learning
    0.8
    EP 398 learning finished: 0.03352456995705211, 0.00430742318905826
    399
    EP 399 simulation finished, start learning
    2.1
    EP 399 learning finished: 0.03199503746477461, 0.0038417914280151344
    400
    EP 400 simulation finished, start learning
    1.9
    EP 400 learning finished: 0.053017141303314474, 0.0041244947821112155
    401
    EP 401 simulation finished, start learning
    1.0
    EP 401 learning finished: 0.033305633401576884, 0.004127306875822412
    402
    EP 402 simulation finished, start learning
    2.1
    EP 402 learning finished: 0.05905096287216338, 0.005556146921204375
    403
    EP 403 simulation finished, start learning
    0.8
    EP 403 learning finished: 0.05817635906482721, 0.005743863031878821
    404
    EP 404 simulation finished, start learning
    -0.5
    EP 404 learning finished: 0.03667410992350639, 0.005332786806914666
    405
    EP 405 simulation finished, start learning
    0.4
    EP 405 learning finished: 0.0364953516087283, 0.004351411312504702
    406
    EP 406 simulation finished, start learning
    -0.1
    EP 406 learning finished: 0.012915292406871615, 0.0029714346275166448
    407
    EP 407 simulation finished, start learning
    0.2
    EP 407 learning finished: 0.012819120447099763, 0.0029788857455156914
    408
    EP 408 simulation finished, start learning
    0.6
    EP 408 learning finished: 0.017341811895478898, 0.002539415469037243
    409
    EP 409 simulation finished, start learning
    -0.1
    EP 409 learning finished: 0.017217936588287408, 0.0028723839054730083
    410
    EP 410 simulation finished, start learning
    -0.3
    EP 410 learning finished: 0.01699998802387892, 0.003120033258059471
    411
    EP 411 simulation finished, start learning
    0.8
    EP 411 learning finished: 0.01679829986418202, 0.003134926705225194
    412
    EP 412 simulation finished, start learning
    -0.3
    EP 412 learning finished: 0.01661240301870137, 0.003104587224555491
    413
    EP 413 simulation finished, start learning
    1.0
    EP 413 learning finished: 0.013825882895719843, 0.0028053580592227872
    414
    EP 414 simulation finished, start learning
    -0.5
    EP 414 learning finished: 0.018892470141203432, 0.002633184601647409
    415
    EP 415 simulation finished, start learning
    0.8
    EP 415 learning finished: 0.02912372042485223, 0.0032824565358949218
    416
    EP 416 simulation finished, start learning
    -0.5
    EP 416 learning finished: 0.028799197051748715, 0.0032939121508208727
    417
    EP 417 simulation finished, start learning
    -0.1
    EP 417 learning finished: 0.037748928947690885, 0.004648939224084596
    418
    EP 418 simulation finished, start learning
    1.0
    EP 418 learning finished: 0.036871046879140223, 0.004185153964612706
    419
    EP 419 simulation finished, start learning
    0.8
    EP 419 learning finished: 0.03611757420530726, 0.0038170779463287674
    420
    EP 420 simulation finished, start learning
    -0.3
    EP 420 learning finished: 0.03532759203264746, 0.003804408829189976
    421
    EP 421 simulation finished, start learning
    0.6
    EP 421 learning finished: 0.024809481749662687, 0.0033278672148468746
    422
    EP 422 simulation finished, start learning
    1.9
    EP 422 learning finished: 0.024373585137446696, 0.003895805242445982
    423
    EP 423 simulation finished, start learning
    -0.3
    EP 423 learning finished: 0.02032585322007563, 0.003922307099525332
    424
    EP 424 simulation finished, start learning
    -0.3
    EP 424 learning finished: 0.06221842184837589, 0.005512381266144797
    425
    EP 425 simulation finished, start learning
    1.7
    EP 425 learning finished: 0.06016498888803001, 0.00467433297432282
    426
    EP 426 simulation finished, start learning
    -0.3
    EP 426 learning finished: 0.05810887205004578, 0.004367742876664291
    427
    EP 427 simulation finished, start learning
    0.8
    EP 427 learning finished: 0.056132070112302035, 0.00415833968124614
    428
    EP 428 simulation finished, start learning
    1.0
    EP 428 learning finished: 0.03895031691509132, 0.003964698092371641
    429
    EP 429 simulation finished, start learning
    1.9
    EP 429 learning finished: 0.03833706967341399, 0.0038487775675884184
    430
    EP 430 simulation finished, start learning
    0.8
    EP 430 learning finished: 0.04563598153327675, 0.0037077692262767192
    431
    EP 431 simulation finished, start learning
    0.8
    EP 431 learning finished: 0.04435220425220621, 0.003998507680681357
    432
    EP 432 simulation finished, start learning
    -0.1
    EP 432 learning finished: 0.01820540117171454, 0.0027823570074716752
    433
    EP 433 simulation finished, start learning
    -0.5
    EP 433 learning finished: 0.03452438390539392, 0.004070384458413817
    434
    EP 434 simulation finished, start learning
    1.0
    EP 434 learning finished: 0.03798504881106268, 0.005111968320485313
    435
    EP 435 simulation finished, start learning
    0.6
    EP 435 learning finished: 0.05692017778170566, 0.006499182479133128
    436
    EP 436 simulation finished, start learning
    1.0
    EP 436 learning finished: 0.05586916149919752, 0.007296632776948933
    437
    EP 437 simulation finished, start learning
    -0.5
    EP 437 learning finished: 0.05485904332319791, 0.006817411971337339
    438
    EP 438 simulation finished, start learning
    -0.1
    EP 438 learning finished: 0.05384465100293134, 0.006459363839506349
    439
    EP 439 simulation finished, start learning
    0.8
    EP 439 learning finished: 0.030862622813288, 0.004622512296397527
    440
    EP 440 simulation finished, start learning
    1.7
    EP 440 learning finished: 0.026568306063045277, 0.0032116990816740965
    441
    EP 441 simulation finished, start learning
    -0.1
    EP 441 learning finished: 0.02927913750457023, 0.0033215060966967774
    442
    EP 442 simulation finished, start learning
    -0.1
    EP 442 learning finished: 0.028663951690186096, 0.002991388864099692
    443
    EP 443 simulation finished, start learning
    2.1
    EP 443 learning finished: 0.028069280690864673, 0.0028119694643942705
    444
    EP 444 simulation finished, start learning
    -0.7
    EP 444 learning finished: 0.027523305332744386, 0.003296057794307722
    445
    EP 445 simulation finished, start learning
    0.8
    EP 445 learning finished: 0.01445136284245585, 0.002430692099487486
    446
    EP 446 simulation finished, start learning
    -0.3
    EP 446 learning finished: 0.05345101325294377, 0.004543929029377815
    447
    EP 447 simulation finished, start learning
    1.0
    EP 447 learning finished: 0.05235981268930234, 0.004524326108768149
    448
    EP 448 simulation finished, start learning
    -0.1
    EP 448 learning finished: 0.05124701982092961, 0.004077830387006841
    449
    EP 449 simulation finished, start learning
    -0.1
    EP 449 learning finished: 0.05022638502291343, 0.005186901073235016
    450
    EP 450 simulation finished, start learning
    1.9
    EP 450 learning finished: 0.03575268997709813, 0.004250450756211617
    451
    EP 451 simulation finished, start learning
    2.1
    EP 451 learning finished: 0.051572079023651796, 0.00584689874187861
    452
    EP 452 simulation finished, start learning
    0.8
    EP 452 learning finished: 0.05082916954119554, 0.00587547113812175
    453
    EP 453 simulation finished, start learning
    -2.0
    EP 453 learning finished: 0.05005314908852121, 0.006354258735536231
    454
    EP 454 simulation finished, start learning
    -0.5
    EP 454 learning finished: 0.0492631470163829, 0.005501372410660954
    455
    EP 455 simulation finished, start learning
    0.8
    EP 455 learning finished: 0.06718317054925205, 0.005134936624626094
    456
    EP 456 simulation finished, start learning
    1.9
    EP 456 learning finished: 0.06443097498458439, 0.005387322411643872
    457
    EP 457 simulation finished, start learning
    0.6
    EP 457 learning finished: 0.031237621382779857, 0.004942940997267969
    458
    EP 458 simulation finished, start learning
    -0.3
    EP 458 learning finished: 0.05097673926811286, 0.005273548999189914
    459
    EP 459 simulation finished, start learning
    0.8
    EP 459 learning finished: 0.021457426725612905, 0.004259201596862189
    460
    EP 460 simulation finished, start learning
    1.0
    EP 460 learning finished: 0.021109240712086752, 0.004053763052905682
    461
    EP 461 simulation finished, start learning
    0.8
    EP 461 learning finished: 0.019241819489591805, 0.003275270697763774
    462
    EP 462 simulation finished, start learning
    0.8
    EP 462 learning finished: 0.011735092260958258, 0.00232947819176825
    463
    EP 463 simulation finished, start learning
    0.6
    EP 463 learning finished: 0.049032618419374785, 0.0038381307541631576
    464
    EP 464 simulation finished, start learning
    -0.3
    EP 464 learning finished: 0.049258162472859635, 0.0048374519418428324
    465
    EP 465 simulation finished, start learning
    0.6
    EP 465 learning finished: 0.04824609346737474, 0.004979095546011218
    466
    EP 466 simulation finished, start learning
    0.8
    EP 466 learning finished: 0.0472481505069855, 0.005539312189221303
    467
    EP 467 simulation finished, start learning
    0.6
    EP 467 learning finished: 0.046281366147003844, 0.004633593655263987
    468
    EP 468 simulation finished, start learning
    1.9
    EP 468 learning finished: 0.035944440329734206, 0.00352527482157812
    469
    EP 469 simulation finished, start learning
    0.8
    EP 469 learning finished: 0.01932863615200553, 0.003165347408881858
    470
    EP 470 simulation finished, start learning
    0.4
    EP 470 learning finished: 0.012587153199291379, 0.0027620028144891007
    471
    EP 471 simulation finished, start learning
    -0.3
    EP 471 learning finished: 0.012061528070443472, 0.0025669862588952597
    472
    EP 472 simulation finished, start learning
    1.0
    EP 472 learning finished: 0.013166075118888898, 0.0026329474436573297
    473
    EP 473 simulation finished, start learning
    0.8
    EP 473 learning finished: 0.013033535701668875, 0.0025372198074599672
    474
    EP 474 simulation finished, start learning
    0.6
    EP 474 learning finished: 0.021821022358575082, 0.002491738919158118
    475
    EP 475 simulation finished, start learning
    -0.5
    EP 475 learning finished: 0.021484968099491297, 0.0024592378810759374
    476
    EP 476 simulation finished, start learning
    0.8
    EP 476 learning finished: 0.021169590125356574, 0.0021113734226358566
    477
    EP 477 simulation finished, start learning
    -0.3
    EP 477 learning finished: 0.014931185947008707, 0.0020019386159084024
    478
    EP 478 simulation finished, start learning
    -0.3
    EP 478 learning finished: 0.014709316403904715, 0.0025840379053267137
    479
    EP 479 simulation finished, start learning
    2.1
    EP 479 learning finished: 0.011990435810520061, 0.0022168347310025927
    480
    EP 480 simulation finished, start learning
    1.9
    EP 480 learning finished: 0.027351614160436855, 0.0029643645678748345
    481
    EP 481 simulation finished, start learning
    2.1
    EP 481 learning finished: 0.027039503210443572, 0.0032820496915791295
    482
    EP 482 simulation finished, start learning
    1.9
    EP 482 learning finished: 0.03140699978927941, 0.003082095989324178
    483
    EP 483 simulation finished, start learning
    1.9
    EP 483 learning finished: 0.06811696972799552, 0.005305910726333317
    484
    EP 484 simulation finished, start learning
    0.6
    EP 484 learning finished: 0.07309868912977535, 0.004749701404955536
    485
    EP 485 simulation finished, start learning
    -0.1
    EP 485 learning finished: 0.07142776712188592, 0.004641788180293621
    486
    EP 486 simulation finished, start learning
    0.8
    EP 486 learning finished: 0.06971025858164026, 0.00430978066988752
    487
    EP 487 simulation finished, start learning
    -0.5
    EP 487 learning finished: 0.014259361436529616, 0.0024662790543054647
    488
    EP 488 simulation finished, start learning
    0.6
    EP 488 learning finished: 0.02019201158630679, 0.0026916868610845635
    489
    EP 489 simulation finished, start learning
    0.8
    EP 489 learning finished: 0.026201553382214637, 0.0030280511836420637
    490
    EP 490 simulation finished, start learning
    -0.7
    EP 490 learning finished: 0.025928475059528333, 0.0032263283708153026
    491
    EP 491 simulation finished, start learning
    1.0
    EP 491 learning finished: 0.05100989585277338, 0.003796779268871911
    492
    EP 492 simulation finished, start learning
    -0.5
    EP 492 learning finished: 0.04988710241098593, 0.003395486220429178
    493
    EP 493 simulation finished, start learning
    -0.1
    EP 493 learning finished: 0.04878377305347925, 0.00335095829684548
    494
    EP 494 simulation finished, start learning
    0.6
    EP 494 learning finished: 0.047753978945670905, 0.00307324570279173
    495
    EP 495 simulation finished, start learning
    2.1
    EP 495 learning finished: 0.023523936162982206, 0.0029157702938138325
    496
    EP 496 simulation finished, start learning
    0.8
    EP 496 learning finished: 0.023266500240624044, 0.0028250622231926342
    497
    EP 497 simulation finished, start learning
    1.9
    EP 497 learning finished: 0.016963433545960527, 0.0022335670077983927
    498
    EP 498 simulation finished, start learning
    -0.5
    EP 498 learning finished: 0.05640371914312059, 0.00398894512664863
    499
    EP 499 simulation finished, start learning
    -0.3
    EP 499 learning finished: 0.054616863606031796, 0.0035129881472579366
    500
    EP 500 simulation finished, start learning
    -0.1
    EP 500 learning finished: 0.05291741471916822, 0.003947709427154847
    501
    EP 501 simulation finished, start learning
    -0.1
    EP 501 learning finished: 0.05121040134113075, 0.004366144289648571
    502
    EP 502 simulation finished, start learning
    -0.3
    EP 502 learning finished: 0.02255795206712019, 0.0027059549188604703
    503
    EP 503 simulation finished, start learning
    1.0
    EP 503 learning finished: 0.02447187999372602, 0.0034127838939493477
    504
    EP 504 simulation finished, start learning
    0.6
    EP 504 learning finished: 0.024218986881283655, 0.0034569094920481463
    505
    EP 505 simulation finished, start learning
    -0.5
    EP 505 learning finished: 0.03774019230654613, 0.004700683967037905
    506
    EP 506 simulation finished, start learning
    -0.3
    EP 506 learning finished: 0.03662061268703365, 0.00493937044297596
    507
    EP 507 simulation finished, start learning
    -0.1
    EP 507 learning finished: 0.03562648023852599, 0.004301523733800217
    508
    EP 508 simulation finished, start learning
    -0.1
    EP 508 learning finished: 0.024584732294509415, 0.004076611895619642
    509
    EP 509 simulation finished, start learning
    0.4
    EP 509 learning finished: 0.024149410997922338, 0.0027433852156949073
    510
    EP 510 simulation finished, start learning
    -0.5
    EP 510 learning finished: 0.04623239326868589, 0.004012465527406722
    511
    EP 511 simulation finished, start learning
    0.1
    EP 511 learning finished: 0.045304836892367195, 0.0036782109863664897
    512
    EP 512 simulation finished, start learning
    1.9
    EP 512 learning finished: 0.04443684390351921, 0.003354680688644543
    513
    EP 513 simulation finished, start learning
    -0.5
    EP 513 learning finished: 0.043756464128730244, 0.005463425226597998
    514
    EP 514 simulation finished, start learning
    0.8
    EP 514 learning finished: 0.04315227730321425, 0.004116348052940172
    515
    EP 515 simulation finished, start learning
    -0.1
    EP 515 learning finished: 0.04256478786310858, 0.004194341346730079
    516
    EP 516 simulation finished, start learning
    1.0
    EP 516 learning finished: 0.041951972657001235, 0.004252402141097733
    517
    EP 517 simulation finished, start learning
    -0.5
    EP 517 learning finished: 0.012629113104300593, 0.002135043183753016
    518
    EP 518 simulation finished, start learning
    2.6
    EP 518 learning finished: 0.07944677349144769, 0.004396063116180904
    519
    EP 519 simulation finished, start learning
    0.1
    EP 519 learning finished: 0.07856151078638263, 0.0044205353458299235
    520
    EP 520 simulation finished, start learning
    0.6
    EP 520 learning finished: 0.07771713042528461, 0.004272365749617674
    521
    EP 521 simulation finished, start learning
    0.8
    EP 521 learning finished: 0.07685864132336206, 0.0048895758514106435
    522
    EP 522 simulation finished, start learning
    2.1
    EP 522 learning finished: 0.02899990385692064, 0.003227409401856368
    523
    EP 523 simulation finished, start learning
    -0.3
    EP 523 learning finished: 0.02857124535272959, 0.003489576868450736
    524
    EP 524 simulation finished, start learning
    0.8
    EP 524 learning finished: 0.028069173396019664, 0.0039008765274525514
    525
    EP 525 simulation finished, start learning
    0.6
    EP 525 learning finished: 0.027774732662020796, 0.0032137632063375943
    526
    EP 526 simulation finished, start learning
    1.0
    EP 526 learning finished: 0.01089628938861864, 0.00247276115111969
    527
    EP 527 simulation finished, start learning
    0.6
    EP 527 learning finished: 0.010798235950945312, 0.0019522639168313705
    528
    EP 528 simulation finished, start learning
    0.6
    EP 528 learning finished: 0.02785789155981888, 0.0023581594874540757
    529
    EP 529 simulation finished, start learning
    -0.3
    EP 529 learning finished: 0.027069393009194087, 0.0026287335086123687
    530
    EP 530 simulation finished, start learning
    -0.3
    EP 530 learning finished: 0.026294181368825775, 0.003179096222402797
    531
    EP 531 simulation finished, start learning
    -0.3
    EP 531 learning finished: 0.02560679935067789, 0.003640383190727889
    532
    EP 532 simulation finished, start learning
    -0.3
    EP 532 learning finished: 0.021997462336700746, 0.003145985570904092
    533
    EP 533 simulation finished, start learning
    1.9
    EP 533 learning finished: 0.02179962423217674, 0.00272010785466667
    534
    EP 534 simulation finished, start learning
    -0.5
    EP 534 learning finished: 0.018759448379542576, 0.002802294003877853
    535
    EP 535 simulation finished, start learning
    1.0
    EP 535 learning finished: 0.041898598581317614, 0.0035377540523695433
    536
    EP 536 simulation finished, start learning
    -1.8
    EP 536 learning finished: 0.050996638395746974, 0.005636392502065615
    537
    EP 537 simulation finished, start learning
    0.6
    EP 537 learning finished: 0.04948333798979743, 0.005874913267438525
    538
    EP 538 simulation finished, start learning
    -0.5
    EP 538 learning finished: 0.04803953388196813, 0.005524229606191464
    539
    EP 539 simulation finished, start learning
    -0.5
    EP 539 learning finished: 0.04664172109249565, 0.005306795712231235
    540
    EP 540 simulation finished, start learning
    0.1
    EP 540 learning finished: 0.021412700086081315, 0.0031919433663400165
    541
    EP 541 simulation finished, start learning
    1.9
    EP 541 learning finished: 0.021215964609460958, 0.0027195196478077555
    542
    EP 542 simulation finished, start learning
    1.9
    EP 542 learning finished: 0.021029662293210016, 0.002674105933198667
    543
    EP 543 simulation finished, start learning
    0.6
    EP 543 learning finished: 0.014932012121590788, 0.0020019516291519546
    544
    EP 544 simulation finished, start learning
    1.9
    EP 544 learning finished: 0.03857965229535054, 0.003066792771519916
    545
    EP 545 simulation finished, start learning
    2.3
    EP 545 learning finished: 0.04184017422109321, 0.003868004259894059
    546
    EP 546 simulation finished, start learning
    0.8
    EP 546 learning finished: 0.03776599080579146, 0.003642555937849151
    547
    EP 547 simulation finished, start learning
    0.6
    EP 547 learning finished: 0.045382144992805795, 0.0049972675747356615
    548
    EP 548 simulation finished, start learning
    -0.1
    EP 548 learning finished: 0.04412218298718873, 0.004095701061576043
    549
    EP 549 simulation finished, start learning
    -1.2
    EP 549 learning finished: 0.04291669326988723, 0.003441310697704078
    550
    EP 550 simulation finished, start learning
    2.1
    EP 550 learning finished: 0.041690502033604915, 0.003074215160968216
    551
    EP 551 simulation finished, start learning
    0.8
    EP 551 learning finished: 0.00859983993482416, 0.0017576296782705059
    552
    EP 552 simulation finished, start learning
    0.6
    EP 552 learning finished: 0.04356475353443656, 0.0027346742501606625
    553
    EP 553 simulation finished, start learning
    0.1
    EP 553 learning finished: 0.04273897927598869, 0.002829188599865126
    554
    EP 554 simulation finished, start learning
    -0.5
    EP 554 learning finished: 0.08315097715709463, 0.004756018898570885
    555
    EP 555 simulation finished, start learning
    -0.3
    EP 555 learning finished: 0.08020218816678604, 0.0053622746967316726
    556
    EP 556 simulation finished, start learning
    0.6
    EP 556 learning finished: 0.03900337525401243, 0.004729711976659626
    557
    EP 557 simulation finished, start learning
    -0.3
    EP 557 learning finished: 0.038359804142387155, 0.005390483889829878
    558
    EP 558 simulation finished, start learning
    1.9
    EP 558 learning finished: 0.04112210573977748, 0.005051001610393235
    559
    EP 559 simulation finished, start learning
    1.0
    EP 559 learning finished: 0.04066693213611927, 0.004362925288736608
    560
    EP 560 simulation finished, start learning
    -0.5
    EP 560 learning finished: 0.04022753546100333, 0.0037272369654761384
    561
    EP 561 simulation finished, start learning
    1.0
    EP 561 learning finished: 0.03979158094901081, 0.0027936904874568125
    562
    EP 562 simulation finished, start learning
    -0.1
    EP 562 learning finished: 0.011808799306768591, 0.0014417664447868295
    563
    EP 563 simulation finished, start learning
    1.2
    EP 563 learning finished: 0.011668390910254622, 0.0013568560926712214
    564
    EP 564 simulation finished, start learning
    -0.3
    EP 564 learning finished: 0.021129342583289912, 0.002006489303158221
    565
    EP 565 simulation finished, start learning
    -0.1
    EP 565 learning finished: 0.020937054420064882, 0.002116582170160114
    566
    EP 566 simulation finished, start learning
    -0.5
    EP 566 learning finished: 0.04132922410926465, 0.003598704945929343
    567
    EP 567 simulation finished, start learning
    0.8
    EP 567 learning finished: 0.04056537998933818, 0.0036812054104684716
    568
    EP 568 simulation finished, start learning
    -0.3
    EP 568 learning finished: 0.020045644736001123, 0.003442848292676312
    569
    EP 569 simulation finished, start learning
    1.9
    EP 569 learning finished: 0.03728234223412663, 0.004446207535674276
    570
    EP 570 simulation finished, start learning
    -0.5
    EP 570 learning finished: 0.04116861188058252, 0.004329474903167346
    571
    EP 571 simulation finished, start learning
    -0.5
    EP 571 learning finished: 0.04009178450620565, 0.004124553793184785
    572
    EP 572 simulation finished, start learning
    -0.1
    EP 572 learning finished: 0.03900978091761953, 0.004517451195652208
    573
    EP 573 simulation finished, start learning
    -0.3
    EP 573 learning finished: 0.03800399091981488, 0.003971259442989816
    574
    EP 574 simulation finished, start learning
    2.3
    EP 574 learning finished: 0.01926182121769715, 0.002406977408338798
    575
    EP 575 simulation finished, start learning
    1.0
    EP 575 learning finished: 0.019102418798431664, 0.002424957209982065
    576
    EP 576 simulation finished, start learning
    -0.3
    EP 576 learning finished: 0.01071305124348676, 0.001988670197177189
    577
    EP 577 simulation finished, start learning
    1.0
    EP 577 learning finished: 0.01068178434550937, 0.001994186937580994
    578
    EP 578 simulation finished, start learning
    -0.1
    EP 578 learning finished: 0.010656132714879973, 0.0023444758386512166
    579
    EP 579 simulation finished, start learning
    0.6
    EP 579 learning finished: 0.009230035050382668, 0.0021362122675785133
    580
    EP 580 simulation finished, start learning
    2.3
    EP 580 learning finished: 0.018264694816857663, 0.001901164631309099
    581
    EP 581 simulation finished, start learning
    0.6
    EP 581 learning finished: 0.01798544083538456, 0.0019567884004226266
    582
    EP 582 simulation finished, start learning
    1.0
    EP 582 learning finished: 0.017702131978446722, 0.0019072883257907625
    583
    EP 583 simulation finished, start learning
    -0.3
    EP 583 learning finished: 0.017414965749406264, 0.0020212400634453676
    584
    EP 584 simulation finished, start learning
    1.9
    EP 584 learning finished: 0.010903737508755285, 0.0022383092962668383
    585
    EP 585 simulation finished, start learning
    -0.3
    EP 585 learning finished: 0.012716503163698434, 0.00237358367392998
    586
    EP 586 simulation finished, start learning
    0.8
    EP 586 learning finished: 0.01631609468941786, 0.0023788678817445452
    587
    EP 587 simulation finished, start learning
    1.9
    EP 587 learning finished: 0.01925837457637769, 0.003062540017132659
    588
    EP 588 simulation finished, start learning
    0.6
    EP 588 learning finished: 0.01913342855603517, 0.002720550056607096
    589
    EP 589 simulation finished, start learning
    0.6
    EP 589 learning finished: 0.0190064109543151, 0.0026351903957791967
    590
    EP 590 simulation finished, start learning
    -0.5
    EP 590 learning finished: 0.03735395201615965, 0.004236903073389947
    591
    EP 591 simulation finished, start learning
    -0.5
    EP 591 learning finished: 0.03638636252358474, 0.0037688508096860307
    592
    EP 592 simulation finished, start learning
    -0.5
    EP 592 learning finished: 0.041268026985855855, 0.004807108630267995
    593
    EP 593 simulation finished, start learning
    3.2
    EP 593 learning finished: 0.039878999004982596, 0.0043958378881359395
    594
    EP 594 simulation finished, start learning
    0.8
    EP 594 learning finished: 0.02351521180518734, 0.0029594999030650437
    595
    EP 595 simulation finished, start learning
    0.6
    EP 595 learning finished: 0.04122953154260145, 0.00390090885245295
    596
    EP 596 simulation finished, start learning
    2.1
    EP 596 learning finished: 0.040372930346425306, 0.0027407013172762174
    597
    EP 597 simulation finished, start learning
    0.8
    EP 597 learning finished: 0.03953585665525239, 0.0031796908719808614
    598
    EP 598 simulation finished, start learning
    0.1
    EP 598 learning finished: 0.03665452025065363, 0.0028575860930531993
    599
    EP 599 simulation finished, start learning
    1.0
    EP 599 learning finished: 0.012413051713256351, 0.0016510123785238584
    600
    EP 600 simulation finished, start learning
    0.6
    EP 600 learning finished: 0.021207570306465584, 0.002328947707993606
    601
    EP 601 simulation finished, start learning
    0.1
    EP 601 learning finished: 0.020706178022843247, 0.0019492308606321174
    602
    EP 602 simulation finished, start learning
    2.1
    EP 602 learning finished: 0.03867637006597935, 0.0029443430293381867
    603
    EP 603 simulation finished, start learning
    -0.1
    EP 603 learning finished: 0.03798443668911533, 0.0031915079809073515
    604
    EP 604 simulation finished, start learning
    1.9
    EP 604 learning finished: 0.03731119128963245, 0.0025427641207210333
    605
    EP 605 simulation finished, start learning
    -0.3
    EP 605 learning finished: 0.03668904858811839, 0.002788464960021858
    606
    EP 606 simulation finished, start learning
    -0.3
    EP 606 learning finished: 0.015084851283032386, 0.0023299079874232465
    607
    EP 607 simulation finished, start learning
    2.1
    EP 607 learning finished: 0.014855888282776064, 0.002184399418624111
    608
    EP 608 simulation finished, start learning
    0.8
    EP 608 learning finished: 0.019488416000708497, 0.002720931560262596
    609
    EP 609 simulation finished, start learning
    -0.3
    EP 609 learning finished: 0.019057503709975165, 0.002499587404778958
    610
    EP 610 simulation finished, start learning
    -0.5
    EP 610 learning finished: 0.019298968964198693, 0.0026270225148792015
    611
    EP 611 simulation finished, start learning
    0.8
    EP 611 learning finished: 0.019132638316668338, 0.0025176821628823537
    612
    EP 612 simulation finished, start learning
    -0.5
    EP 612 learning finished: 0.01895707301054106, 0.002181911489711308
    613
    EP 613 simulation finished, start learning
    0.8
    EP 613 learning finished: 0.01878398556924843, 0.0026365953738635983
    614
    EP 614 simulation finished, start learning
    -0.5
    EP 614 learning finished: 0.017497075626126346, 0.0020918567891937967
    615
    EP 615 simulation finished, start learning
    1.0
    EP 615 learning finished: 0.021896766055458095, 0.0025753105261661913
    616
    EP 616 simulation finished, start learning
    -0.5
    EP 616 learning finished: 0.02156431098393874, 0.0025072562017544875
    617
    EP 617 simulation finished, start learning
    0.6
    EP 617 learning finished: 0.021269550806155557, 0.0021514201935857458
    618
    EP 618 simulation finished, start learning
    -0.5
    EP 618 learning finished: 0.02096673042363406, 0.002361165293118528
    619
    EP 619 simulation finished, start learning
    0.6
    EP 619 learning finished: 0.03619116419605328, 0.0028653217272980452
    620
    EP 620 simulation finished, start learning
    1.2
    EP 620 learning finished: 0.03557748913859715, 0.0026904399797245404
    621
    EP 621 simulation finished, start learning
    0.6
    EP 621 learning finished: 0.03500075368553235, 0.0028800711888708613
    622
    EP 622 simulation finished, start learning
    -0.3
    EP 622 learning finished: 0.03439729241509193, 0.0026728493052713548
    623
    EP 623 simulation finished, start learning
    1.0
    EP 623 learning finished: 0.008367288434186548, 0.0016363089555138377
    624
    EP 624 simulation finished, start learning
    0.6
    EP 624 learning finished: 0.016561028535280187, 0.0018431697662790458
    625
    EP 625 simulation finished, start learning
    0.6
    EP 625 learning finished: 0.016516697837001715, 0.001980449388502198
    626
    EP 626 simulation finished, start learning
    -0.5
    EP 626 learning finished: 0.016172031972978118, 0.0020659958047527793
    627
    EP 627 simulation finished, start learning
    0.8
    EP 627 learning finished: 0.015838248141415434, 0.0021467491326825072
    628
    EP 628 simulation finished, start learning
    -0.9
    EP 628 learning finished: 0.03381961182538307, 0.0035919888106886316
    629
    EP 629 simulation finished, start learning
    -0.5
    EP 629 learning finished: 0.03326685651355854, 0.0034248985352288985
    630
    EP 630 simulation finished, start learning
    1.0
    EP 630 learning finished: 0.03270774968139767, 0.0033150110763598695
    631
    EP 631 simulation finished, start learning
    -0.3
    EP 631 learning finished: 0.032172665202411466, 0.003433121134659836
    632
    EP 632 simulation finished, start learning
    -0.5
    EP 632 learning finished: 0.027515262460724443, 0.0022501324005445578
    633
    EP 633 simulation finished, start learning
    0.6
    EP 633 learning finished: 0.034204530905052266, 0.0038272856450916604
    634
    EP 634 simulation finished, start learning
    1.0
    EP 634 learning finished: 0.033813208759955415, 0.003856054601046044
    635
    EP 635 simulation finished, start learning
    -0.5
    EP 635 learning finished: 0.03347551980546015, 0.003806995169403707
    636
    EP 636 simulation finished, start learning
    2.1
    EP 636 learning finished: 0.03311264728362673, 0.003313426757196619
    637
    EP 637 simulation finished, start learning
    -0.5
    EP 637 learning finished: 0.00914167745973149, 0.0014368293652311237
    638
    EP 638 simulation finished, start learning
    2.3
    EP 638 learning finished: 0.009046690672352753, 0.0012340965171338258
    639
    EP 639 simulation finished, start learning
    -0.5
    EP 639 learning finished: 0.008958040129794398, 0.001321260786315781
    640
    EP 640 simulation finished, start learning
    0.4
    EP 640 learning finished: 0.00886318448105694, 0.0013956963615364693
    641
    EP 641 simulation finished, start learning
    0.6
    EP 641 learning finished: 0.03289777882442202, 0.0030204098499426735
    642
    EP 642 simulation finished, start learning
    -0.5
    EP 642 learning finished: 0.03251518941712561, 0.0035012104034150004
    643
    EP 643 simulation finished, start learning
    1.0
    EP 643 learning finished: 0.032158103287789785, 0.0032485266921878364
    644
    EP 644 simulation finished, start learning
    -0.3
    EP 644 learning finished: 0.031782175163062476, 0.0038426897464139548
    645
    EP 645 simulation finished, start learning
    -0.1
    EP 645 learning finished: 0.01742408283960284, 0.002241593901948405
    646
    EP 646 simulation finished, start learning
    -0.5
    EP 646 learning finished: 0.03411876421777382, 0.0024358910206634665
    647
    EP 647 simulation finished, start learning
    0.6
    EP 647 learning finished: 0.0335507954375216, 0.002748578811257521
    648
    EP 648 simulation finished, start learning
    1.7
    EP 648 learning finished: 0.016561338054720043, 0.0021335091169658454
    649
    EP 649 simulation finished, start learning
    2.3
    EP 649 learning finished: 0.0227752036316069, 0.002889066356281722
    650
    EP 650 simulation finished, start learning
    -0.1
    EP 650 learning finished: 0.02264343168257099, 0.0024291134739905518
    651
    EP 651 simulation finished, start learning
    1.0
    EP 651 learning finished: 0.022508898654319154, 0.0026101778023174538
    652
    EP 652 simulation finished, start learning
    0.6
    EP 652 learning finished: 0.022372770366914096, 0.0027117143030252755
    653
    EP 653 simulation finished, start learning
    1.2
    EP 653 learning finished: 0.019395514783033363, 0.002040853005487453
    654
    EP 654 simulation finished, start learning
    0.8
    EP 654 learning finished: 0.019136361767546184, 0.001969359567735529
    655
    EP 655 simulation finished, start learning
    -0.1
    EP 655 learning finished: 0.015519221266874439, 0.0018391393025894154
    656
    EP 656 simulation finished, start learning
    -0.1
    EP 656 learning finished: 0.021127435944604755, 0.001954573642580698
    657
    EP 657 simulation finished, start learning
    2.1
    EP 657 learning finished: 0.020524351937474705, 0.0017427949942257215
    658
    EP 658 simulation finished, start learning
    -0.5
    EP 658 learning finished: 0.019899868888373362, 0.0021729086215262344
    659
    EP 659 simulation finished, start learning
    1.0
    EP 659 learning finished: 0.019333776656353052, 0.0026728801419974857
    660
    EP 660 simulation finished, start learning
    0.8
    EP 660 learning finished: 0.019147192258817114, 0.0026061852823394543
    661
    EP 661 simulation finished, start learning
    -0.3
    EP 661 learning finished: 0.03545037777365234, 0.0041023770469340165
    662
    EP 662 simulation finished, start learning
    2.1
    EP 662 learning finished: 0.03457714005957335, 0.0035543844568058647
    663
    EP 663 simulation finished, start learning
    -0.5
    EP 663 learning finished: 0.03373624058003788, 0.0031097387034218493
    664
    EP 664 simulation finished, start learning
    0.4
    EP 664 learning finished: 0.032951482955155154, 0.003371056924604446
    665
    EP 665 simulation finished, start learning
    -0.3
    EP 665 learning finished: 0.01565235560890521, 0.0020575384284917952
    666
    EP 666 simulation finished, start learning
    1.0
    EP 666 learning finished: 0.018167233548290085, 0.0030331313300392404
    667
    EP 667 simulation finished, start learning
    -0.5
    EP 667 learning finished: 0.017928578387926386, 0.0028624303590197853
    668
    EP 668 simulation finished, start learning
    0.6
    EP 668 learning finished: 0.03058994807412141, 0.0032661725311047274
    669
    EP 669 simulation finished, start learning
    1.0
    EP 669 learning finished: 0.030319021121348655, 0.003298158570358811
    670
    EP 670 simulation finished, start learning
    1.0
    EP 670 learning finished: 0.030057352993345887, 0.002275226175518452
    671
    EP 671 simulation finished, start learning
    0.8
    EP 671 learning finished: 0.03242859724362157, 0.0031269186242911112
    672
    EP 672 simulation finished, start learning
    -0.5
    EP 672 learning finished: 0.03224265279404781, 0.0035452803454895515
    673
    EP 673 simulation finished, start learning
    2.3
    EP 673 learning finished: 0.031732127686961764, 0.0033171703967094235
    674
    EP 674 simulation finished, start learning
    1.0
    EP 674 learning finished: 0.03123401272011339, 0.0035593120458185384
    675
    EP 675 simulation finished, start learning
    1.0
    EP 675 learning finished: 0.03075673144964597, 0.0027217412082823494
    676
    EP 676 simulation finished, start learning
    0.8
    EP 676 learning finished: 0.010281599332204294, 0.0015116942666505429
    677
    EP 677 simulation finished, start learning
    0.8
    EP 677 learning finished: 0.010222253634140088, 0.001703293057740085
    678
    EP 678 simulation finished, start learning
    0.4
    EP 678 learning finished: 0.008717066582750732, 0.001344214424406548
    679
    EP 679 simulation finished, start learning
    0.4
    EP 679 learning finished: 0.00861243136686518, 0.0013513023600571217
    680
    EP 680 simulation finished, start learning
    0.8
    EP 680 learning finished: 0.031210425834709454, 0.0021936096184444753
    681
    EP 681 simulation finished, start learning
    1.0
    EP 681 learning finished: 0.030882238263709283, 0.0021937677060974714
    682
    EP 682 simulation finished, start learning
    0.4
    EP 682 learning finished: 0.030550306103993563, 0.0021411671580406355
    683
    EP 683 simulation finished, start learning
    1.0
    EP 683 learning finished: 0.0254313637163186, 0.0018073706752857466
    684
    EP 684 simulation finished, start learning
    0.8
    EP 684 learning finished: 0.029737071000688342, 0.0022375519348738077
    685
    EP 685 simulation finished, start learning
    -0.1
    EP 685 learning finished: 0.029072494749902145, 0.002194492536748291
    686
    EP 686 simulation finished, start learning
    0.8
    EP 686 learning finished: 0.028419180261140653, 0.0025407540881852986
    687
    EP 687 simulation finished, start learning
    -0.5
    EP 687 learning finished: 0.027798655757027113, 0.0024719328175707433
    688
    EP 688 simulation finished, start learning
    1.9
    EP 688 learning finished: 0.011426468275933743, 0.001593270110158418
    689
    EP 689 simulation finished, start learning
    2.3
    EP 689 learning finished: 0.007901959518854085, 0.0018648006879816437
    690
    EP 690 simulation finished, start learning
    0.4
    EP 690 learning finished: 0.007840893840556973, 0.001829866479380049
    691
    EP 691 simulation finished, start learning
    -0.3
    EP 691 learning finished: 0.030457454784593274, 0.00266458611233944
    692
    EP 692 simulation finished, start learning
    -0.3
    EP 692 learning finished: 0.02998248336975262, 0.0035445742384536127
    693
    EP 693 simulation finished, start learning
    0.6
    EP 693 learning finished: 0.02952712480263653, 0.003256293788764436
    694
    EP 694 simulation finished, start learning
    -0.3
    EP 694 learning finished: 0.029074095521675503, 0.002971912067755683
    695
    EP 695 simulation finished, start learning
    1.0
    EP 695 learning finished: 0.02551218912808828, 0.0020387228358327847
    696
    EP 696 simulation finished, start learning
    1.9
    EP 696 learning finished: 0.006737180766725359, 0.0011034966997785778
    697
    EP 697 simulation finished, start learning
    0.8
    EP 697 learning finished: 0.007031714457877314, 0.0009669310877509774
    698
    EP 698 simulation finished, start learning
    0.4
    EP 698 learning finished: 0.010459842199098901, 0.0010932347779653573
    699
    EP 699 simulation finished, start learning
    -0.3
    EP 699 learning finished: 0.010364809930162089, 0.0013086797061745148
    700
    EP 700 simulation finished, start learning
    -0.3
    EP 700 learning finished: 0.017448605074642742, 0.0021447879156828165
    701
    EP 701 simulation finished, start learning
    0.8
    EP 701 learning finished: 0.02779110442581434, 0.002966882196529491
    702
    EP 702 simulation finished, start learning
    0.4
    EP 702 learning finished: 0.02755668764373098, 0.002896947602234412
    703
    EP 703 simulation finished, start learning
    2.1
    EP 703 learning finished: 0.027341292448225474, 0.002798792482830631
    704
    EP 704 simulation finished, start learning
    1.0
    EP 704 learning finished: 0.0271493778647649, 0.0020041235686946157
    705
    EP 705 simulation finished, start learning
    0.6
    EP 705 learning finished: 0.006787377964723618, 0.0012417582010588735
    706
    EP 706 simulation finished, start learning
    0.8
    EP 706 learning finished: 0.0287470650301489, 0.002035218656980989
    707
    EP 707 simulation finished, start learning
    1.7
    EP 707 learning finished: 0.02832104484280551, 0.0025142269981169446
    708
    EP 708 simulation finished, start learning
    0.6
    EP 708 learning finished: 0.02791276998639436, 0.002499922102592894
    709
    EP 709 simulation finished, start learning
    0.4
    EP 709 learning finished: 0.027497801503991504, 0.002533926560045502
    710
    EP 710 simulation finished, start learning
    0.8
    EP 710 learning finished: 0.02976208566306937, 0.002764445848270236
    711
    EP 711 simulation finished, start learning
    1.0
    EP 711 learning finished: 0.029460204318459304, 0.002157170810139303
    712
    EP 712 simulation finished, start learning
    -0.1
    EP 712 learning finished: 0.029167057014099385, 0.0023599470451291793
    713
    EP 713 simulation finished, start learning
    -0.5
    EP 713 learning finished: 0.02401040317752745, 0.002486157337413225
    714
    EP 714 simulation finished, start learning
    0.4
    EP 714 learning finished: 0.027051917216558863, 0.0026245014091310637
    715
    EP 715 simulation finished, start learning
    0.6
    EP 715 learning finished: 0.026833261334299863, 0.002725395871122166
    716
    EP 716 simulation finished, start learning
    -0.3
    EP 716 learning finished: 0.026615741607564303, 0.003245927439227192
    717
    EP 717 simulation finished, start learning
    -0.3
    EP 717 learning finished: 0.02639792838644439, 0.002790887336992648
    718
    EP 718 simulation finished, start learning
    0.6
    EP 718 learning finished: 0.02417370532692953, 0.0020680229708665288
    719
    EP 719 simulation finished, start learning
    -0.1
    EP 719 learning finished: 0.023700245374936202, 0.0021687825526314763
    720
    EP 720 simulation finished, start learning
    1.9
    EP 720 learning finished: 0.00729296421058212, 0.0015022393553465463
    721
    EP 721 simulation finished, start learning
    2.1
    EP 721 learning finished: 0.00777203456531339, 0.0014582215617131022
    722
    EP 722 simulation finished, start learning
    0.6
    EP 722 learning finished: 0.008358192480532267, 0.001324890902662551
    723
    EP 723 simulation finished, start learning
    1.0
    EP 723 learning finished: 0.010844780434868362, 0.0014667695669893233
    724
    EP 724 simulation finished, start learning
    -0.1
    EP 724 learning finished: 0.015648068827271988, 0.0019073468962021028
    725
    EP 725 simulation finished, start learning
    -0.3
    EP 725 learning finished: 0.015516572450572585, 0.0022497351788817593
    726
    EP 726 simulation finished, start learning
    -0.5
    EP 726 learning finished: 0.021095501156714214, 0.0028305807611091055
    727
    EP 727 simulation finished, start learning
    -0.3
    EP 727 learning finished: 0.015282974486875212, 0.00248861704774081
    728
    EP 728 simulation finished, start learning
    -0.1
    EP 728 learning finished: 0.02057574412793528, 0.002322163401116545
    729
    EP 729 simulation finished, start learning
    0.8
    EP 729 learning finished: 0.020237598420452585, 0.0022008784360200706
    730
    EP 730 simulation finished, start learning
    0.6
    EP 730 learning finished: 0.01991630113887588, 0.0019320269464627726
    731
    EP 731 simulation finished, start learning
    -0.5
    EP 731 learning finished: 0.01960310334260562, 0.002181737841472492
    732
    EP 732 simulation finished, start learning
    1.0
    EP 732 learning finished: 0.011878885758596525, 0.0019100486203954825
    733
    EP 733 simulation finished, start learning
    0.8
    EP 733 learning finished: 0.01949756241103806, 0.002026037823132376
    734
    EP 734 simulation finished, start learning
    -0.5
    EP 734 learning finished: 0.018944127197856986, 0.0019193649503493337
    735
    EP 735 simulation finished, start learning
    0.8
    EP 735 learning finished: 0.01435734925751575, 0.00203697592236593
    736
    EP 736 simulation finished, start learning
    1.9
    EP 736 learning finished: 0.031770877834802036, 0.0028420464455096717
    737
    EP 737 simulation finished, start learning
    0.8
    EP 737 learning finished: 0.03149797150311073, 0.0025082945561103187
    738
    EP 738 simulation finished, start learning
    -0.3
    EP 738 learning finished: 0.03121662543533077, 0.0028136147751061658
    739
    EP 739 simulation finished, start learning
    0.4
    EP 739 learning finished: 0.030951076205184336, 0.003406861397470956
    740
    EP 740 simulation finished, start learning
    -0.3
    EP 740 learning finished: 0.023081155319475855, 0.002371349671041939
    741
    EP 741 simulation finished, start learning
    0.1
    EP 741 learning finished: 0.02266166960318472, 0.0024579242601620588
    742
    EP 742 simulation finished, start learning
    -1.4
    EP 742 learning finished: 0.044113265343425745, 0.0030125579218774345
    743
    EP 743 simulation finished, start learning
    2.1
    EP 743 learning finished: 0.021447452425736202, 0.001996159739811626
    744
    EP 744 simulation finished, start learning
    1.0
    EP 744 learning finished: 0.021101198812240263, 0.0021061538787081058
    745
    EP 745 simulation finished, start learning
    -0.1
    EP 745 learning finished: 0.020720629323803585, 0.0020691352070065228
    746
    EP 746 simulation finished, start learning
    1.0
    EP 746 learning finished: 0.007253588302429392, 0.001195822704846361
    747
    EP 747 simulation finished, start learning
    -0.3
    EP 747 learning finished: 0.009347075725192333, 0.0014731611385815635
    748
    EP 748 simulation finished, start learning
    2.1
    EP 748 learning finished: 0.009285661399913714, 0.0012733844614321033
    749
    EP 749 simulation finished, start learning
    -0.3
    EP 749 learning finished: 0.009224952680365384, 0.001356005953209366
    750
    EP 750 simulation finished, start learning
    -0.5
    EP 750 learning finished: 0.009149953065078265, 0.001438133303917427
    751
    EP 751 simulation finished, start learning
    0.6
    EP 751 learning finished: 0.008007064386792884, 0.001372421317337419
    752
    EP 752 simulation finished, start learning
    0.6
    EP 752 learning finished: 0.00840865833829696, 0.0014437963390332709
    753
    EP 753 simulation finished, start learning
    0.8
    EP 753 learning finished: 0.008296005175917998, 0.0013591971300287442
    754
    EP 754 simulation finished, start learning
    0.1
    EP 754 learning finished: 0.008183194390655402, 0.0010895900354083998
    755
    EP 755 simulation finished, start learning
    -0.3
    EP 755 learning finished: 0.005655444508223084, 0.0010511735048019666
    756
    EP 756 simulation finished, start learning
    -0.1
    EP 756 learning finished: 0.007057326568619793, 0.0012618312964887135
    757
    EP 757 simulation finished, start learning
    -0.3
    EP 757 learning finished: 0.009149702888637812, 0.001605011292164893
    758
    EP 758 simulation finished, start learning
    0.8
    EP 758 learning finished: 0.009075914962116904, 0.0016095935133196293
    759
    EP 759 simulation finished, start learning
    0.6
    EP 759 learning finished: 0.009010271908010736, 0.0016600416260438566
    760
    EP 760 simulation finished, start learning
    1.0
    EP 760 learning finished: 0.008938189732746515, 0.0016375927598202278
    761
    EP 761 simulation finished, start learning
    -0.7
    EP 761 learning finished: 0.020641513920317323, 0.00218521380772127
    762
    EP 762 simulation finished, start learning
    -0.1
    EP 762 learning finished: 0.020259263662533833, 0.0020307390399414427
    763
    EP 763 simulation finished, start learning
    -0.3
    EP 763 learning finished: 0.01992058151507692, 0.001966978964110161
    764
    EP 764 simulation finished, start learning
    -0.3
    EP 764 learning finished: 0.019566559964470542, 0.0022475889181206715
    765
    EP 765 simulation finished, start learning
    0.8
    EP 765 learning finished: 0.015197280947349956, 0.0016059521233532797
    766
    EP 766 simulation finished, start learning
    -0.1
    EP 766 learning finished: 0.030029172432208373, 0.002037087937718382
    767
    EP 767 simulation finished, start learning
    0.8
    EP 767 learning finished: 0.04416881929298855, 0.0023487377634748424
    768
    EP 768 simulation finished, start learning
    -0.3
    EP 768 learning finished: 0.0288748841083879, 0.002460149594856173
    769
    EP 769 simulation finished, start learning
    -0.5
    EP 769 learning finished: 0.04250288387578749, 0.0038337995518758217
    770
    EP 770 simulation finished, start learning
    0.6
    EP 770 learning finished: 0.02779196835025477, 0.003859406919017802
    771
    EP 771 simulation finished, start learning
    -0.1
    EP 771 learning finished: 0.026770579294528307, 0.003322476664524801
    772
    EP 772 simulation finished, start learning
    0.8
    EP 772 learning finished: 0.026406277834566616, 0.0029453809993608276
    773
    EP 773 simulation finished, start learning
    -0.1
    EP 773 learning finished: 0.013110659743970743, 0.0015840544858597982
    774
    EP 774 simulation finished, start learning
    -0.3
    EP 774 learning finished: 0.013582943387817181, 0.0015704493448865375
    775
    EP 775 simulation finished, start learning
    0.6
    EP 775 learning finished: 0.013485297944205943, 0.0016761097591627278
    776
    EP 776 simulation finished, start learning
    -0.1
    EP 776 learning finished: 0.013392577481187828, 0.0017612012338225624
    777
    EP 777 simulation finished, start learning
    1.9
    EP 777 learning finished: 0.013300902535634762, 0.0015597925560524306
    778
    EP 778 simulation finished, start learning
    0.8
    EP 778 learning finished: 0.008807839812005724, 0.001165182694751965
    779
    EP 779 simulation finished, start learning
    -0.1
    EP 779 learning finished: 0.008741984640073142, 0.001203722652309197
    780
    EP 780 simulation finished, start learning
    1.0
    EP 780 learning finished: 0.005263235259455712, 0.0010734618927939754
    781
    EP 781 simulation finished, start learning
    -0.9
    EP 781 learning finished: 0.005369144474126841, 0.0011186120524339094
    782
    EP 782 simulation finished, start learning
    0.1
    EP 782 learning finished: 0.007451213880975693, 0.0011504641831044937
    783
    EP 783 simulation finished, start learning
    0.8
    EP 783 learning finished: 0.026215170808836508, 0.00185042830027575
    784
    EP 784 simulation finished, start learning
    -0.5
    EP 784 learning finished: 0.025856466114863252, 0.0019251807617233219
    785
    EP 785 simulation finished, start learning
    0.8
    EP 785 learning finished: 0.025491448468375033, 0.002231705704275999
    786
    EP 786 simulation finished, start learning
    0.6
    EP 786 learning finished: 0.02516892557327921, 0.002532827456750597
    787
    EP 787 simulation finished, start learning
    -0.1
    EP 787 learning finished: 0.026762935623681372, 0.0021667576898207475
    788
    EP 788 simulation finished, start learning
    -0.7
    EP 788 learning finished: 0.025817060828462246, 0.003132548938469884
    789
    EP 789 simulation finished, start learning
    0.4
    EP 789 learning finished: 0.024489554835135152, 0.003093413075832885
    790
    EP 790 simulation finished, start learning
    2.1
    EP 790 learning finished: 0.02415379390904171, 0.002816534723181302
    791
    EP 791 simulation finished, start learning
    1.0
    EP 791 learning finished: 0.023838389233413082, 0.0023120304495389266
    792
    EP 792 simulation finished, start learning
    0.8
    EP 792 learning finished: 0.009910960629824217, 0.0014585278845012928
    793
    EP 793 simulation finished, start learning
    -0.3
    EP 793 learning finished: 0.00873861368694051, 0.0013072303315327938
    794
    EP 794 simulation finished, start learning
    0.6
    EP 794 learning finished: 0.00960840404424168, 0.0013883737137211916
    795
    EP 795 simulation finished, start learning
    -0.5
    EP 795 learning finished: 0.009542690145060817, 0.0016777929147816784
    796
    EP 796 simulation finished, start learning
    1.2
    EP 796 learning finished: 0.008665607063151448, 0.0013262787121890163
    797
    EP 797 simulation finished, start learning
    0.8
    EP 797 learning finished: 0.017139921385525447, 0.0014759380423002566
    798
    EP 798 simulation finished, start learning
    0.8
    EP 798 learning finished: 0.016906608720826277, 0.0015711426373847537
    799
    EP 799 simulation finished, start learning
    1.0
    EP 799 learning finished: 0.008366204683109224, 0.001229004738205447
    800
    EP 800 simulation finished, start learning
    0.6
    EP 800 learning finished: 0.008527821616221232, 0.0014699581241432724
    801
    EP 801 simulation finished, start learning
    -0.5
    EP 801 learning finished: 0.02359533512196421, 0.001967305960316079
    802
    EP 802 simulation finished, start learning
    1.7
    EP 802 learning finished: 0.02328514582680352, 0.0018182215454549955
    803
    EP 803 simulation finished, start learning
    1.0
    EP 803 learning finished: 0.022987728304249, 0.0022931729480341195
    804
    EP 804 simulation finished, start learning
    0.4
    EP 804 learning finished: 0.022696276367474955, 0.002373792673221173
    805
    EP 805 simulation finished, start learning
    0.8
    EP 805 learning finished: 0.013177030032088588, 0.0016650794527558188
    806
    EP 806 simulation finished, start learning
    -0.3
    EP 806 learning finished: 0.026069959981416257, 0.0019349625249022312
    807
    EP 807 simulation finished, start learning
    0.8
    EP 807 learning finished: 0.01289505508637001, 0.001483447643841826
    808
    EP 808 simulation finished, start learning
    1.9
    EP 808 learning finished: 0.02694018593853187, 0.0021743965093507234
    809
    EP 809 simulation finished, start learning
    1.0
    EP 809 learning finished: 0.02673339883892467, 0.0023176270288671613
    810
    EP 810 simulation finished, start learning
    0.8
    EP 810 learning finished: 0.026529049365048962, 0.001994647303840358
    811
    EP 811 simulation finished, start learning
    1.9
    EP 811 learning finished: 0.07866959866304413, 0.0037254299515058764
    812
    EP 812 simulation finished, start learning
    0.6
    EP 812 learning finished: 0.05193118822185738, 0.003003458785735158
    813
    EP 813 simulation finished, start learning
    0.6
    EP 813 learning finished: 0.051540933880104944, 0.0030955192256860165
    814
    EP 814 simulation finished, start learning
    1.0
    EP 814 learning finished: 0.05115428409460776, 0.0031800622565044956
    815
    EP 815 simulation finished, start learning
    0.8
    EP 815 learning finished: 0.008144588301715494, 0.0014814683317281307
    816
    EP 816 simulation finished, start learning
    1.0
    EP 816 learning finished: 0.00808990151086153, 0.001472728413429325
    817
    EP 817 simulation finished, start learning
    0.8
    EP 817 learning finished: 0.006487712217899677, 0.0011938237372443317
    818
    EP 818 simulation finished, start learning
    1.0
    EP 818 learning finished: 0.006448482827194901, 0.0012245088897886447
    819
    EP 819 simulation finished, start learning
    -0.5
    EP 819 learning finished: 0.006399395027004928, 0.001310315519908234
    820
    EP 820 simulation finished, start learning
    0.8
    EP 820 learning finished: 0.010940898335005222, 0.0014638025925532983
    821
    EP 821 simulation finished, start learning
    0.8
    EP 821 learning finished: 0.014687318324826393, 0.0019203111672325135
    822
    EP 822 simulation finished, start learning
    1.9
    EP 822 learning finished: 0.014450271238185763, 0.0019491171145365749
    823
    EP 823 simulation finished, start learning
    0.6
    EP 823 learning finished: 0.0142193598336533, 0.001983751703456438
    824
    EP 824 simulation finished, start learning
    0.6
    EP 824 learning finished: 0.014010464125934874, 0.0017486604779203274
    825
    EP 825 simulation finished, start learning
    1.9
    EP 825 learning finished: 0.008925377875319995, 0.00127915117162003
    826
    EP 826 simulation finished, start learning
    -0.5
    EP 826 learning finished: 0.022539307433050215, 0.0017778823049271059
    827
    EP 827 simulation finished, start learning
    0.6
    EP 827 learning finished: 0.0222449492729746, 0.0017648920782361854
    828
    EP 828 simulation finished, start learning
    0.8
    EP 828 learning finished: 0.021956436761520592, 0.0017385986063060965
    829
    EP 829 simulation finished, start learning
    -0.1
    EP 829 learning finished: 0.021679685168655816, 0.001812386898750462
    830
    EP 830 simulation finished, start learning
    -0.3
    EP 830 learning finished: 0.00805044702911939, 0.001462134198495918
    831
    EP 831 simulation finished, start learning
    1.0
    EP 831 learning finished: 0.008012938130235625, 0.0014858932083334775
    832
    EP 832 simulation finished, start learning
    -0.3
    EP 832 learning finished: 0.007959766862458473, 0.0016587808651274127
    833
    EP 833 simulation finished, start learning
    1.0
    EP 833 learning finished: 0.007913540413022169, 0.0017868479229866396
    834
    EP 834 simulation finished, start learning
    -0.3
    EP 834 learning finished: 0.007855137900748765, 0.0016731971658330846
    835
    EP 835 simulation finished, start learning
    0.8
    EP 835 learning finished: 0.008073305654907159, 0.0017853049755131023
    836
    EP 836 simulation finished, start learning
    0.1
    EP 836 learning finished: 0.008036820568555925, 0.0016318759993187715
    837
    EP 837 simulation finished, start learning
    -0.5
    EP 837 learning finished: 0.015379989163056607, 0.001750859630100715
    838
    EP 838 simulation finished, start learning
    -0.3
    EP 838 learning finished: 0.012730845129733215, 0.0018461751851678448
    839
    EP 839 simulation finished, start learning
    -0.3
    EP 839 learning finished: 0.02518657322136164, 0.0019672556937995467
    840
    EP 840 simulation finished, start learning
    -0.3
    EP 840 learning finished: 0.037120448774381476, 0.00224224661981072
    841
    EP 841 simulation finished, start learning
    -0.5
    EP 841 learning finished: 0.03635583083208571, 0.0019628500437786217
    842
    EP 842 simulation finished, start learning
    0.8
    EP 842 learning finished: 0.023815880772056985, 0.001939484163227534
    843
    EP 843 simulation finished, start learning
    1.0
    EP 843 learning finished: 0.011786837657071114, 0.0019287717205487415
    844
    EP 844 simulation finished, start learning
    1.2
    EP 844 learning finished: 0.01041051244972202, 0.001793371079094469
    845
    EP 845 simulation finished, start learning
    0.8
    EP 845 learning finished: 0.010306891549420882, 0.0017680765879846128
    846
    EP 846 simulation finished, start learning
    -0.3
    EP 846 learning finished: 0.02157534999970423, 0.0021091486771107892
    847
    EP 847 simulation finished, start learning
    -0.1
    EP 847 learning finished: 0.02130100821377301, 0.0020170766198625845
    848
    EP 848 simulation finished, start learning
    0.8
    EP 848 learning finished: 0.021031971366859636, 0.0019278498653496068
    849
    EP 849 simulation finished, start learning
    0.8
    EP 849 learning finished: 0.02077387893467275, 0.0020409074264767235
    850
    EP 850 simulation finished, start learning
    1.0
    EP 850 learning finished: 0.010127514449212782, 0.0016662736323128369
    851
    EP 851 simulation finished, start learning
    0.8
    EP 851 learning finished: 0.010037150106201764, 0.0014709807764659855
    852
    EP 852 simulation finished, start learning
    0.8
    EP 852 learning finished: 0.009928684476271599, 0.0013777225475548327
    853
    EP 853 simulation finished, start learning
    1.0
    EP 853 learning finished: 0.0057298660824294245, 0.001261804466476668
    854
    EP 854 simulation finished, start learning
    1.9
    EP 854 learning finished: 0.02377339664679745, 0.001790510483170804
    855
    EP 855 simulation finished, start learning
    -0.1
    EP 855 learning finished: 0.023623088381842905, 0.002338624345856119
    856
    EP 856 simulation finished, start learning
    -0.5
    EP 856 learning finished: 0.023460604406256635, 0.0029187376126413866
    857
    EP 857 simulation finished, start learning
    -0.1
    EP 857 learning finished: 0.02329844839706574, 0.0030258547030558997
    858
    EP 858 simulation finished, start learning
    0.6
    EP 858 learning finished: 0.019822867139164035, 0.0026282807653454365
    859
    EP 859 simulation finished, start learning
    -0.5
    EP 859 learning finished: 0.01675124152517249, 0.002252566921059379
    860
    EP 860 simulation finished, start learning
    1.9
    EP 860 learning finished: 0.016306772252050905, 0.001873979596386438
    861
    EP 861 simulation finished, start learning
    3.0
    EP 861 learning finished: 0.05006280842043864, 0.0032493014998052627
    862
    EP 862 simulation finished, start learning
    1.0
    EP 862 learning finished: 0.049702779806963804, 0.003531605306147184
    863
    EP 863 simulation finished, start learning
    0.6
    EP 863 learning finished: 0.04935168788386335, 0.0034401931451902845
    864
    EP 864 simulation finished, start learning
    0.6
    EP 864 learning finished: 0.05073140506463858, 0.003940302755556291
    865
    EP 865 simulation finished, start learning
    0.6
    EP 865 learning finished: 0.024480636323810145, 0.0027003767950219415
    866
    EP 866 simulation finished, start learning
    0.8
    EP 866 learning finished: 0.022878425229150068, 0.002232375207517401
    867
    EP 867 simulation finished, start learning
    -0.5
    EP 867 learning finished: 0.022719547276171426, 0.002138613615179752
    868
    EP 868 simulation finished, start learning
    1.0
    EP 868 learning finished: 0.017869087625365765, 0.001393806195550712
    869
    EP 869 simulation finished, start learning
    -0.5
    EP 869 learning finished: 0.008803620782931176, 0.0011043577442594321
    870
    EP 870 simulation finished, start learning
    -0.3
    EP 870 learning finished: 0.00692294982479158, 0.0010992048197794035
    871
    EP 871 simulation finished, start learning
    0.1
    EP 871 learning finished: 0.010271309241570492, 0.0009478530271165853
    872
    EP 872 simulation finished, start learning
    0.8
    EP 872 learning finished: 0.010106581568384065, 0.000983214003288907
    873
    EP 873 simulation finished, start learning
    0.8
    EP 873 learning finished: 0.009971129959485125, 0.0010077764857704397
    874
    EP 874 simulation finished, start learning
    2.3
    EP 874 learning finished: 0.007723969271856035, 0.0010957486201087496
    875
    EP 875 simulation finished, start learning
    0.8
    EP 875 learning finished: 0.007691582609281156, 0.0010671297989613138
    876
    EP 876 simulation finished, start learning
    1.0
    EP 876 learning finished: 0.007010264386464016, 0.0010303431232661087
    877
    EP 877 simulation finished, start learning
    0.8
    EP 877 learning finished: 0.013796025401479284, 0.0013206475280735097
    878
    EP 878 simulation finished, start learning
    -0.3
    EP 878 learning finished: 0.019620308999371172, 0.0016868549464714583
    879
    EP 879 simulation finished, start learning
    -0.3
    EP 879 learning finished: 0.019399999168967952, 0.002427181687667923
    880
    EP 880 simulation finished, start learning
    0.6
    EP 880 learning finished: 0.019178251343839214, 0.0025542356292255486
    881
    EP 881 simulation finished, start learning
    -0.1
    EP 881 learning finished: 0.018955134573560795, 0.0023417621973224195
    882
    EP 882 simulation finished, start learning
    1.0
    EP 882 learning finished: 0.0168759512660257, 0.001714616148275866
    883
    EP 883 simulation finished, start learning
    0.8
    EP 883 learning finished: 0.010031581852729232, 0.0010768965999057856
    884
    EP 884 simulation finished, start learning
    3.2
    EP 884 learning finished: 0.03494792368745436, 0.0019141402871579727
    885
    EP 885 simulation finished, start learning
    0.6
    EP 885 learning finished: 0.03606540726906715, 0.0019203003896122173
    886
    EP 886 simulation finished, start learning
    0.6
    EP 886 learning finished: 0.0355867491514239, 0.0018809866816985599
    887
    EP 887 simulation finished, start learning
    -0.3
    EP 887 learning finished: 0.035110467354520836, 0.002026813964267876
    888
    EP 888 simulation finished, start learning
    -0.1
    EP 888 learning finished: 0.004899293388895387, 0.0010180309420412537
    889
    EP 889 simulation finished, start learning
    0.6
    EP 889 learning finished: 0.0048791059375412615, 0.001155244139273398
    890
    EP 890 simulation finished, start learning
    0.6
    EP 890 learning finished: 0.009283032834712657, 0.0011302682904306138
    891
    EP 891 simulation finished, start learning
    -0.3
    EP 891 learning finished: 0.00918518155345005, 0.0010661239818338941
    892
    EP 892 simulation finished, start learning
    1.9
    EP 892 learning finished: 0.00908572165436361, 0.0012572051216771518
    893
    EP 893 simulation finished, start learning
    0.6
    EP 893 learning finished: 0.008756094615074872, 0.0011716085507950114
    894
    EP 894 simulation finished, start learning
    0.1
    EP 894 learning finished: 0.008668060069672023, 0.001179735959431022
    895
    EP 895 simulation finished, start learning
    1.0
    EP 895 learning finished: 0.008595152179362131, 0.001164528633968931
    896
    EP 896 simulation finished, start learning
    0.8
    EP 896 learning finished: 0.007544304910485522, 0.0011681150709566546
    897
    EP 897 simulation finished, start learning
    1.0
    EP 897 learning finished: 0.007496695213686344, 0.0012012588961925752
    898
    EP 898 simulation finished, start learning
    -0.3
    EP 898 learning finished: 0.008956061624580514, 0.001344162165075713
    899
    EP 899 simulation finished, start learning
    1.0
    EP 899 learning finished: 0.008869682120263178, 0.0013126609629779962
    900
    EP 900 simulation finished, start learning
    1.9
    EP 900 learning finished: 0.008790986159466385, 0.0010688296143755993
    901
    EP 901 simulation finished, start learning
    -0.1
    EP 901 learning finished: 0.011577076423830324, 0.0014174167264843635
    902
    EP 902 simulation finished, start learning
    -0.3
    EP 902 learning finished: 0.011499136673211563, 0.0013299399446116272
    903
    EP 903 simulation finished, start learning
    -0.3
    EP 903 learning finished: 0.022770684902851723, 0.0018064179905226269
    904
    EP 904 simulation finished, start learning
    0.1
    EP 904 learning finished: 0.022474423699994084, 0.0017836057888777068
    905
    EP 905 simulation finished, start learning
    0.8
    EP 905 learning finished: 0.011124320418187228, 0.0015374712019485398
    906
    EP 906 simulation finished, start learning
    0.1
    EP 906 learning finished: 0.01105401129868211, 0.0015129381364738098
    907
    EP 907 simulation finished, start learning
    0.8
    EP 907 learning finished: 0.008603920702215717, 0.0011587125312533253
    908
    EP 908 simulation finished, start learning
    0.6
    EP 908 learning finished: 0.010984412629875262, 0.0014279435749710995
    909
    EP 909 simulation finished, start learning
    -0.1
    EP 909 learning finished: 0.010915514775547752, 0.0014212942015775082
    910
    EP 910 simulation finished, start learning
    -0.3
    EP 910 learning finished: 0.018945968255510426, 0.0019356070268896908
    911
    EP 911 simulation finished, start learning
    -0.5
    EP 911 learning finished: 0.018722840081781378, 0.002093052826458386
    912
    EP 912 simulation finished, start learning
    -0.3
    EP 912 learning finished: 0.01851391523512458, 0.0019947941141331693
    913
    EP 913 simulation finished, start learning
    1.9
    EP 913 learning finished: 0.018308676834936755, 0.0018298139534209945
    914
    EP 914 simulation finished, start learning
    -0.1
    EP 914 learning finished: 0.00728957974395783, 0.0014046800338496646
    915
    EP 915 simulation finished, start learning
    0.8
    EP 915 learning finished: 0.00827645224337381, 0.001229852131983511
    916
    EP 916 simulation finished, start learning
    0.4
    EP 916 learning finished: 0.019592651755840684, 0.0017553697617819775
    917
    EP 917 simulation finished, start learning
    -0.3
    EP 917 learning finished: 0.019238878272625826, 0.0017783664063238391
    918
    EP 918 simulation finished, start learning
    0.6
    EP 918 learning finished: 0.018923068718573965, 0.0017177122558951763
    919
    EP 919 simulation finished, start learning
    -0.3
    EP 919 learning finished: 0.01858666638551476, 0.001955991374246596
    920
    EP 920 simulation finished, start learning
    1.0
    EP 920 learning finished: 0.005196717871716494, 0.001197490151700259
    921
    EP 921 simulation finished, start learning
    -0.3
    EP 921 learning finished: 0.006062203070698047, 0.0013808182668460234
    922
    EP 922 simulation finished, start learning
    -0.3
    EP 922 learning finished: 0.006621630316923088, 0.0012665237403809981
    923
    EP 923 simulation finished, start learning
    1.0
    EP 923 learning finished: 0.00657969813464554, 0.0009687443332331014
    924
    EP 924 simulation finished, start learning
    0.6
    EP 924 learning finished: 0.0065441618912744914, 0.0009667271762053842
    925
    EP 925 simulation finished, start learning
    1.0
    EP 925 learning finished: 0.004092074329973272, 0.0008480744034030374
    926
    EP 926 simulation finished, start learning
    -0.3
    EP 926 learning finished: 0.004080465603141903, 0.0008787495651897403
    927
    EP 927 simulation finished, start learning
    -0.1
    EP 927 learning finished: 0.006277498300958584, 0.0009632492581095331
    928
    EP 928 simulation finished, start learning
    -0.3
    EP 928 learning finished: 0.02150122829181278, 0.0015893843258448226
    929
    EP 929 simulation finished, start learning
    0.6
    EP 929 learning finished: 0.021222894324927744, 0.001518269165872388
    930
    EP 930 simulation finished, start learning
    -0.5
    EP 930 learning finished: 0.031324154766043666, 0.0018451342766755002
    931
    EP 931 simulation finished, start learning
    0.6
    EP 931 learning finished: 0.030731716151607635, 0.0020192301333253615
    932
    EP 932 simulation finished, start learning
    -0.3
    EP 932 learning finished: 0.010115853247030238, 0.0017313856228108049
    933
    EP 933 simulation finished, start learning
    1.7
    EP 933 learning finished: 0.010060942287791974, 0.0017014239759368943
    934
    EP 934 simulation finished, start learning
    -0.5
    EP 934 learning finished: 0.009531306462168487, 0.0013665761932843412
    935
    EP 935 simulation finished, start learning
    -0.5
    EP 935 learning finished: 0.008050734592620401, 0.0011600971078480904
    936
    EP 936 simulation finished, start learning
    -0.3
    EP 936 learning finished: 0.010009211949721042, 0.0012973294716947803
    937
    EP 937 simulation finished, start learning
    -0.3
    EP 937 learning finished: 0.019830628904723646, 0.001593872457799822
    938
    EP 938 simulation finished, start learning
    -0.3
    EP 938 learning finished: 0.019593001345853622, 0.0016258584820310618
    939
    EP 939 simulation finished, start learning
    -0.3
    EP 939 learning finished: 0.019365080671260415, 0.0021259442447509047
    940
    EP 940 simulation finished, start learning
    0.8
    EP 940 learning finished: 0.0168420615957654, 0.001812267187418115
    941
    EP 941 simulation finished, start learning
    -0.3
    EP 941 learning finished: 0.016658403045785697, 0.001546463163848738
    942
    EP 942 simulation finished, start learning
    1.9
    EP 942 learning finished: 0.016482042736466695, 0.0014142331940482644
    943
    EP 943 simulation finished, start learning
    -0.7
    EP 943 learning finished: 0.0047161702000568795, 0.001043615552340199
    944
    EP 944 simulation finished, start learning
    -0.3
    EP 944 learning finished: 0.006303278719499872, 0.0009338259085345801
    945
    EP 945 simulation finished, start learning
    0.8
    EP 945 learning finished: 0.009528217161546948, 0.001048248167628163
    946
    EP 946 simulation finished, start learning
    0.8
    EP 946 learning finished: 0.016308298077113648, 0.0015638036874615346
    947
    EP 947 simulation finished, start learning
    -0.3
    EP 947 learning finished: 0.016098732774811886, 0.0013940629605571075
    948
    EP 948 simulation finished, start learning
    1.7
    EP 948 learning finished: 0.01590894610316429, 0.001475735916855292
    949
    EP 949 simulation finished, start learning
    -0.3
    EP 949 learning finished: 0.015718782766323436, 0.0014983136769301242
    950
    EP 950 simulation finished, start learning
    -0.3
    EP 950 learning finished: 0.007331406755046288, 0.001113923564564051
    951
    EP 951 simulation finished, start learning
    -0.1
    EP 951 learning finished: 0.009655835723456185, 0.0015827563293193403
    952
    EP 952 simulation finished, start learning
    -0.5
    EP 952 learning finished: 0.009557074648074426, 0.0015366749408757263
    953
    EP 953 simulation finished, start learning
    0.1
    EP 953 learning finished: 0.009446980054379495, 0.0014326040626106984
    954
    EP 954 simulation finished, start learning
    1.0
    EP 954 learning finished: 0.009351832557624284, 0.001599805535125065
    955
    EP 955 simulation finished, start learning
    1.0
    EP 955 learning finished: 0.007166115597323497, 0.0011937888039039724
    956
    EP 956 simulation finished, start learning
    -0.3
    EP 956 learning finished: 0.018719863553600113, 0.0018435057850897663
    957
    EP 957 simulation finished, start learning
    0.6
    EP 957 learning finished: 0.01843199443551402, 0.0015965549256683198
    958
    EP 958 simulation finished, start learning
    1.0
    EP 958 learning finished: 0.01814523685373004, 0.0015100269281047018
    959
    EP 959 simulation finished, start learning
    -0.3
    EP 959 learning finished: 0.017847773954488666, 0.0019346739892716347
    960
    EP 960 simulation finished, start learning
    -0.3
    EP 960 learning finished: 0.009479500178739997, 0.0015179576323683386
    961
    EP 961 simulation finished, start learning
    0.6
    EP 961 learning finished: 0.00940476336305629, 0.001436273546359057
    962
    EP 962 simulation finished, start learning
    -0.3
    EP 962 learning finished: 0.009326120424283602, 0.001776911169095563
    963
    EP 963 simulation finished, start learning
    0.8
    EP 963 learning finished: 0.009208618328051976, 0.0014248938520160237
    964
    EP 964 simulation finished, start learning
    0.8
    EP 964 learning finished: 0.009109351937651056, 0.0012207642610749248
    965
    EP 965 simulation finished, start learning
    0.6
    EP 965 learning finished: 0.02355224121595434, 0.001718402645252498
    966
    EP 966 simulation finished, start learning
    0.4
    EP 966 learning finished: 0.02043019401014412, 0.0020367404867549233
    967
    EP 967 simulation finished, start learning
    2.3
    EP 967 learning finished: 0.02016318536253081, 0.0018228500688889093
    968
    EP 968 simulation finished, start learning
    0.8
    EP 968 learning finished: 0.02173012670711927, 0.0021139010180343254
    969
    EP 969 simulation finished, start learning
    -0.1
    EP 969 learning finished: 0.021584804692622583, 0.0016935754485585264
    970
    EP 970 simulation finished, start learning
    0.8
    EP 970 learning finished: 0.00913410348654331, 0.0010840140002237875
    971
    EP 971 simulation finished, start learning
    0.6
    EP 971 learning finished: 0.009068510949282071, 0.001190791960798715
    972
    EP 972 simulation finished, start learning
    1.0
    EP 972 learning finished: 0.019403307231332434, 0.001498233506606772
    973
    EP 973 simulation finished, start learning
    -0.3
    EP 973 learning finished: 0.019140713446573088, 0.0018005128833441242
    974
    EP 974 simulation finished, start learning
    1.7
    EP 974 learning finished: 0.01888705164881621, 0.0018359310753070235
    975
    EP 975 simulation finished, start learning
    0.8
    EP 975 learning finished: 0.020739160687888614, 0.0019135062294732552
    976
    EP 976 simulation finished, start learning
    1.0
    EP 976 learning finished: 0.009236655068527, 0.001357314491674172
    977
    EP 977 simulation finished, start learning
    -0.3
    EP 977 learning finished: 0.006011823549219031, 0.001036302176228715
    978
    EP 978 simulation finished, start learning
    1.7
    EP 978 learning finished: 0.005970084657453967, 0.0009320811491934089
    979
    EP 979 simulation finished, start learning
    -0.1
    EP 979 learning finished: 0.006932761590836556, 0.0010382627243564202
    980
    EP 980 simulation finished, start learning
    1.0
    EP 980 learning finished: 0.006903522637853321, 0.0009041550563339273
    981
    EP 981 simulation finished, start learning
    -0.5
    EP 981 learning finished: 0.013680960992357605, 0.0013393477617404115
    982
    EP 982 simulation finished, start learning
    0.6
    EP 982 learning finished: 0.013517106459147499, 0.0014583423833463838
    983
    EP 983 simulation finished, start learning
    1.0
    EP 983 learning finished: 0.009089331462765138, 0.001588552841719909
    984
    EP 984 simulation finished, start learning
    -0.3
    EP 984 learning finished: 0.009038703207910892, 0.0017248058438097125
    985
    EP 985 simulation finished, start learning
    -0.1
    EP 985 learning finished: 0.011905120087631005, 0.0014246078980920673
    986
    EP 986 simulation finished, start learning
    1.0
    EP 986 learning finished: 0.017612773703662654, 0.0014435889186005801
    987
    EP 987 simulation finished, start learning
    1.2
    EP 987 learning finished: 0.017322079854430683, 0.00144781566628302
    988
    EP 988 simulation finished, start learning
    -0.3
    EP 988 learning finished: 0.017870949204875863, 0.0020229311849548903
    989
    EP 989 simulation finished, start learning
    -0.5
    EP 989 learning finished: 0.017582708088668575, 0.002364332417110676
    990
    EP 990 simulation finished, start learning
    0.8
    EP 990 learning finished: 0.017307019412306168, 0.002284219768011131
    991
    EP 991 simulation finished, start learning
    -0.3
    EP 991 learning finished: 0.017043472216340838, 0.002023047178140447
    992
    EP 992 simulation finished, start learning
    0.8
    EP 992 learning finished: 0.010677732387581074, 0.0013325710177791439
    993
    EP 993 simulation finished, start learning
    -0.3
    EP 993 learning finished: 0.006172725476689678, 0.001085681880718059
    994
    EP 994 simulation finished, start learning
    0.4
    EP 994 learning finished: 0.007858641571347746, 0.0011921677151704724
    995
    EP 995 simulation finished, start learning
    1.0
    EP 995 learning finished: 0.007788862610173197, 0.0013123537867149038
    996
    EP 996 simulation finished, start learning
    -0.7
    EP 996 learning finished: 0.012828244001188871, 0.0016649997553881968
    997
    EP 997 simulation finished, start learning
    -0.3
    EP 997 learning finished: 0.012671798036529403, 0.0014538245048675291
    998
    EP 998 simulation finished, start learning
    -0.5
    EP 998 learning finished: 0.01253137085242706, 0.0014186378105778275
    999
    EP 999 simulation finished, start learning
    -0.3
    EP 999 learning finished: 0.009075523019837384, 0.0015343195499906703



```python
v_model, pi_model= q_model.q.yield_v_pi()
```


```python
v_model
```




    0    10.503215
    1    13.631626
    2    17.004731
    3    20.501046
    4    24.007436
    5    17.109083
    6    13.948660
    7    10.778401
    dtype: float64




```python
display_pi(pi_model)
```

    
        -----   -----   -----
        | d |   | u |   | d |
        ---------------------
        | r | r | u | l | l |
        ---------------------
        



```python

```


```python
for s in state_space:
    for a in control_space:
        print(f'state: {s}, action_buffer: {[a]}, action to take: {pi_model((s,a))}')
        
def pi_model(i):
    s, a = i
    return q_model.q.max_value_act(model_model(s, [a]))[1]
```

    state: 0, action_buffer: [0], action to take: 1
    state: 0, action_buffer: [1], action to take: 3
    state: 0, action_buffer: [2], action to take: 1
    state: 0, action_buffer: [3], action to take: 1
    state: 1, action_buffer: [0], action to take: 1
    state: 1, action_buffer: [1], action to take: 3
    state: 1, action_buffer: [2], action to take: 3
    state: 1, action_buffer: [3], action to take: 3
    state: 2, action_buffer: [0], action to take: 3
    state: 2, action_buffer: [1], action to take: 3
    state: 2, action_buffer: [2], action to take: 3
    state: 2, action_buffer: [3], action to take: 0
    state: 3, action_buffer: [0], action to take: 0
    state: 3, action_buffer: [1], action to take: 0
    state: 3, action_buffer: [2], action to take: 3
    state: 3, action_buffer: [3], action to take: 2
    state: 4, action_buffer: [0], action to take: 1
    state: 4, action_buffer: [1], action to take: 0
    state: 4, action_buffer: [2], action to take: 0
    state: 4, action_buffer: [3], action to take: 0
    state: 5, action_buffer: [0], action to take: 2
    state: 5, action_buffer: [1], action to take: 2
    state: 5, action_buffer: [2], action to take: 0
    state: 5, action_buffer: [3], action to take: 2
    state: 6, action_buffer: [0], action to take: 1
    state: 6, action_buffer: [1], action to take: 2
    state: 6, action_buffer: [2], action to take: 2
    state: 6, action_buffer: [3], action to take: 2
    state: 7, action_buffer: [0], action to take: 1
    state: 7, action_buffer: [1], action to take: 2
    state: 7, action_buffer: [2], action to take: 1
    state: 7, action_buffer: [3], action to take: 1



```python
def pi_wrapper(pi):
    def ret(x):
        return pi[x]
    return ret

def eval_pi_model(env, pi, model, discount_factor, MAX_ITER = 1000000):

    path_len = int(np.log(1e-3)/np.log(discount_factor))
    path_num = int(MAX_ITER/path_len)
    obs = 0
    disc_arr = np.array([discount_factor**n for n in range(path_len)])
    r_ls = np.array([[np.nan]*path_len for _ in range(path_num)])
    ret = []
    delay_step = env.num
    action_buffer = [-1]*delay_step
    for k in range(path_num):
        obs = env.reset(random= True)
        for i in range(path_len):
            action = pi(model(obs, action_buffer))
            action_buffer.pop(0)
            action_buffer.append(action)
            next_obs, reward, _, _ = env.step(action)
    #         discounted_cum_r = reward + discount_factor*discounted_cum_r
            r_ls[k, i] = reward
            obs = next_obs
        
    return np.mean(r_ls), r_ls.mean(axis = 1)

def cummean(arr):
    return np.cumsum(arr)/np.arange(1, len(arr)+1)
```


```python
val, v_samples_lin = eval_pi_model(env, pi_wrapper(pi_l), model_model, 0.95)
val
```




    4.875468543105966




```python
exp2_model = v_samples_lin
```


```python
sns.distplot(v_samples_lin, label = 'model')
plt.legend()
```


```python
val, v_samples_lin = eval_pi_model(env, lambda x: np.random.choice([-1,1]), model_model, 0.95)
val
```




    -0.0023362149317737234




```python
exp2_random = v_samples_lin
```


```python
sns.distplot(v_samples_lin, label = 'random benchmark')
plt.legend()
```


```python
# Q-learning using model, async reward collection and optimal action selection



class Q_learning:
    def __init__(self, q, env, delay_step, model, epsilon = 0.1, MAX_ITER = 10000, episode_length = 100, threshold = 1e-3):
        self.q = q
        self.env = env
        self.delay_step = delay_step
        self.epsilon = epsilon
        self.MAX_ITER = MAX_ITER
        self.episode_length = episode_length
        self.threshold = threshold
        self.pool = Pool(400)
        self.qs = []
        self.action_memory = list(np.random.choice(control_space, delay_step))
        self.model = model
        
    def train(self, lr = lambda *args: 0.1):
        q = self.q
        env = self.env
        epsilon = self.epsilon
        MAX_ITER = self.MAX_ITER
        episode_length = self.episode_length
        threshold = self.threshold
        model = self.model
        for i in range(MAX_ITER):
            print(i)
            obs = env.reset(random = True)
            r_ls = []
            for _ in range(100):
                
                
                action, is_random = (np.random.choice(control_space), True) \
                                    if np.random.uniform() < epsilon else \
                                    (get_opt_act(q, model, obs, self.action_memory), False)
                last_action = self.action_memory.pop(0)
                self.action_memory.append(action)
                next_obs, reward, _, _ = env.step(action)
                self.pool.append((obs, next_obs, last_action, reward, is_random))
                r_ls.append(reward)
                obs = next_obs
            if not self.pool.isFull:
                continue
            print(f'EP {i} simulation finished, start learning')
            q_new = self.update_q(q, self.pool ,i, 0.95, lr)
            self.q = q_new
            print(np.mean(r_ls))
            self.qs.append(q_new)
            print(f'EP {i} learning finished: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
            if q_new.difference(q).L_1() < threshold and False:
                break
            else:
                q = q_new
        return q_new
    
    
    def update_q(self, q, pool, t ,alpha, lr):
        pool = pool.shuffle()
        for _ in range(1):
            q = deepcopy(q)
#           
            for s, next_s, a, r, is_random in pool:
#                 if is_random: continue
                q[s, a] = q[s,a] + lr(t, s, a)*(r + alpha*q.max_value_act(next_s)[0]-q[s,a])
#             print(f'Learning ep: {i}: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
#             if q_new.difference(q).L_1() < self.threshold:
#                 break
#             else:
#                 q = q_new
        return q

def get_opt_act(q, model, x, acts):
    act_idx = np.argmax([np.sum([q[s, u]*model(s, x, acts) for s in q.state_space]) for u in q.control_space])
    return q.control_space[act_idx]
```


```python
# P(s|x, a_1, a_2, ...)
def model(s, x, acts):
    return trans_prob(s, x, acts[0])
```


```python
class Learning_Rate:
    def __init__(self, state_space, control_space):
        self.counter = dict(
                zip(list(product(state_space, control_space)), [0]*len(state_space)*len(control_space))
        )
    def __call__(self, i, s, a):
        self.counter[(s, a)] += 1
        return 1/self.counter[(s, a)]
lr_fn = Learning_Rate(state_space, control_space)
# lr_fn = lambda i,s,a: 1/(i+1)
# lr_fn = lambda i,s,a: 0.1
env = delayed_wrapper(Env(), 1)
q = Q_table(random = False)
q_opt = Q_learning(q, env, 1, model, MAX_ITER = 1000)
learned_q = q_opt.train(lr_fn)
```

    0
    1
    2
    3
    EP 3 simulation finished, start learning
    -2.85
    EP 3 learning finished: 10.37109375, 1.9924665901272007
    4
    EP 4 simulation finished, start learning
    -1.52
    EP 4 learning finished: 3.9305127243699625, 0.8749702325860466
    5
    EP 5 simulation finished, start learning
    0.83
    EP 5 learning finished: 9.131421781825246, 1.5315219664049922
    6
    EP 6 simulation finished, start learning
    1.23
    EP 6 learning finished: 9.097810332532, 1.04337686510367
    7
    EP 7 simulation finished, start learning
    0.84
    EP 7 learning finished: 2.0163150736707216, 0.7433366629683249
    8
    EP 8 simulation finished, start learning
    0.9
    EP 8 learning finished: 1.0281981925751094, 0.4840168113472542
    9
    EP 9 simulation finished, start learning
    0.84
    EP 9 learning finished: 0.8200583854334056, 0.3447088779711177
    10
    EP 10 simulation finished, start learning
    1.01
    EP 10 learning finished: 7.743090620867068, 0.5167592166787699
    11
    EP 11 simulation finished, start learning
    0.51
    EP 11 learning finished: 0.6431122426428644, 0.21671055591808977
    12
    EP 12 simulation finished, start learning
    1.32
    EP 12 learning finished: 0.516015514764204, 0.19250820592062373
    13
    EP 13 simulation finished, start learning
    0.91
    EP 13 learning finished: 0.4509848735290154, 0.16234687460908337
    14
    EP 14 simulation finished, start learning
    0.86
    EP 14 learning finished: 0.7022171456266566, 0.1521655007514514
    15
    EP 15 simulation finished, start learning
    1.06
    EP 15 learning finished: 0.28434974964797455, 0.12251155165835084
    16
    EP 16 simulation finished, start learning
    0.98
    EP 16 learning finished: 0.3030492431953169, 0.1096136721401308
    17
    EP 17 simulation finished, start learning
    0.86
    EP 17 learning finished: 0.9028973358703727, 0.13722029634897753
    18
    EP 18 simulation finished, start learning
    0.84
    EP 18 learning finished: 0.6854681364438111, 0.12386017174163783
    19
    EP 19 simulation finished, start learning
    1.05
    EP 19 learning finished: 0.5312969273050783, 0.12872158574713538
    20
    EP 20 simulation finished, start learning
    0.92
    EP 20 learning finished: 0.49878308116414183, 0.124826957910576
    21
    EP 21 simulation finished, start learning
    0.9
    EP 21 learning finished: 0.43080824522349137, 0.12015719069102479
    22
    EP 22 simulation finished, start learning
    0.75
    EP 22 learning finished: 0.442322108331592, 0.11118322703052454
    23
    EP 23 simulation finished, start learning
    1.12
    EP 23 learning finished: 0.34301963295383153, 0.09413355388755087
    24
    EP 24 simulation finished, start learning
    0.96
    EP 24 learning finished: 0.3302691325346281, 0.0845209629594221
    25
    EP 25 simulation finished, start learning
    0.75
    EP 25 learning finished: 0.26884542948991585, 0.06776590440027719
    26
    EP 26 simulation finished, start learning
    1.16
    EP 26 learning finished: 0.1931973244951375, 0.06128782609382999
    27
    EP 27 simulation finished, start learning
    1.36
    EP 27 learning finished: 0.18544150153092298, 0.055652050256718745
    28
    EP 28 simulation finished, start learning
    0.7
    EP 28 learning finished: 0.23396842777425642, 0.05820860181810023
    29
    EP 29 simulation finished, start learning
    0.46
    EP 29 learning finished: 0.20326992680025135, 0.06268559613990363
    30
    EP 30 simulation finished, start learning
    1.01
    EP 30 learning finished: 0.17614667282850238, 0.059990118179686955
    31
    EP 31 simulation finished, start learning
    0.62
    EP 31 learning finished: 0.23546828399049946, 0.06826012224572639
    32
    EP 32 simulation finished, start learning
    0.94
    EP 32 learning finished: 0.4220836622862979, 0.06540690276890673
    33
    EP 33 simulation finished, start learning
    0.89
    EP 33 learning finished: 0.5426811806541488, 0.06833057192722886
    34
    EP 34 simulation finished, start learning
    1.43
    EP 34 learning finished: 0.5841914876676466, 0.07380865704814071
    35
    EP 35 simulation finished, start learning
    1.18
    EP 35 learning finished: 0.3625477410360758, 0.06720757978122566
    36
    EP 36 simulation finished, start learning
    1.47
    EP 36 learning finished: 0.215073177620283, 0.061434811391531244
    37
    EP 37 simulation finished, start learning
    0.84
    EP 37 learning finished: 0.20931653250635573, 0.05637884518779614
    38
    EP 38 simulation finished, start learning
    1.31
    EP 38 learning finished: 0.18439959390363914, 0.04746738401684737
    39
    EP 39 simulation finished, start learning
    0.67
    EP 39 learning finished: 0.23924697038285148, 0.05235610913058896
    40
    EP 40 simulation finished, start learning
    0.75
    EP 40 learning finished: 0.32904119195999537, 0.06331151709177801
    41
    EP 41 simulation finished, start learning
    1.09
    EP 41 learning finished: 0.28316203415539587, 0.054820521015655516
    42
    EP 42 simulation finished, start learning
    0.9
    EP 42 learning finished: 0.2517543137563427, 0.05764417748413998
    43
    EP 43 simulation finished, start learning
    0.92
    EP 43 learning finished: 0.1899108452319891, 0.04922527919109884
    44
    EP 44 simulation finished, start learning
    1.03
    EP 44 learning finished: 0.15792029674352515, 0.04318840074314161
    45
    EP 45 simulation finished, start learning
    1.1
    EP 45 learning finished: 0.1335345608979448, 0.04444752175983374
    46
    EP 46 simulation finished, start learning
    1.28
    EP 46 learning finished: 0.17184946920199984, 0.040527586210947306
    47
    EP 47 simulation finished, start learning
    0.8
    EP 47 learning finished: 0.1905112519350478, 0.038442783475130075
    48
    EP 48 simulation finished, start learning
    1.22
    EP 48 learning finished: 0.13651064150315761, 0.03114943539153113
    49
    EP 49 simulation finished, start learning
    0.87
    EP 49 learning finished: 0.183406544346584, 0.029770753296993163
    50
    EP 50 simulation finished, start learning
    1.08
    EP 50 learning finished: 0.08672862290743222, 0.025335436196165143
    51
    EP 51 simulation finished, start learning
    0.78
    EP 51 learning finished: 0.11107925593887646, 0.02780327511177861
    52
    EP 52 simulation finished, start learning
    0.96
    EP 52 learning finished: 0.10164867672892264, 0.028744242505031638
    53
    EP 53 simulation finished, start learning
    1.14
    EP 53 learning finished: 0.12884483260433832, 0.02962567552809673
    54
    EP 54 simulation finished, start learning
    0.83
    EP 54 learning finished: 0.12000810923345417, 0.0349825893049794
    55
    EP 55 simulation finished, start learning
    1.05
    EP 55 learning finished: 0.15010262882890713, 0.032056466786975206
    56
    EP 56 simulation finished, start learning
    1.24
    EP 56 learning finished: 0.13942282170592524, 0.035128359908644935
    57
    EP 57 simulation finished, start learning
    1.03
    EP 57 learning finished: 0.13067061843462824, 0.032881881162589284
    58
    EP 58 simulation finished, start learning
    1.12
    EP 58 learning finished: 0.16001628550947444, 0.030948855022428973
    59
    EP 59 simulation finished, start learning
    0.99
    EP 59 learning finished: 0.15298580698715458, 0.03055295313734835
    60
    EP 60 simulation finished, start learning
    0.99
    EP 60 learning finished: 0.09713956429021131, 0.02992147481166245
    61
    EP 61 simulation finished, start learning
    0.87
    EP 61 learning finished: 0.07267618181753477, 0.02831600948342608
    62
    EP 62 simulation finished, start learning
    1.18
    EP 62 learning finished: 0.10272821641413943, 0.028107042789006748
    63
    EP 63 simulation finished, start learning
    0.95
    EP 63 learning finished: 0.08998351764946788, 0.027081373148134108
    64
    EP 64 simulation finished, start learning
    1.22
    EP 64 learning finished: 0.08759340072101729, 0.022362383344537384
    65
    EP 65 simulation finished, start learning
    1.0
    EP 65 learning finished: 0.11298894931809134, 0.02292902614093663
    66
    EP 66 simulation finished, start learning
    1.01
    EP 66 learning finished: 0.05407680400174408, 0.01946345989942541
    67
    EP 67 simulation finished, start learning
    1.27
    EP 67 learning finished: 0.05284642311670762, 0.019400596354624114
    68
    EP 68 simulation finished, start learning
    0.96
    EP 68 learning finished: 0.19759384167547767, 0.025629297807979998
    69
    EP 69 simulation finished, start learning
    0.87
    EP 69 learning finished: 0.13503131849018635, 0.02656826424895105
    70
    EP 70 simulation finished, start learning
    0.98
    EP 70 learning finished: 0.1258847941749508, 0.025140304047633774
    71
    EP 71 simulation finished, start learning
    1.31
    EP 71 learning finished: 0.1176753761289211, 0.023923443586157528
    72
    EP 72 simulation finished, start learning
    1.12
    EP 72 learning finished: 0.06044600982600734, 0.018289183134606396
    73
    EP 73 simulation finished, start learning
    0.7
    EP 73 learning finished: 0.0389289128023087, 0.015308112519053785
    74
    EP 74 simulation finished, start learning
    0.86
    EP 74 learning finished: 0.07429504717097313, 0.020259182629050923
    75
    EP 75 simulation finished, start learning
    1.1
    EP 75 learning finished: 0.07285461953428451, 0.022695845837027946
    76
    EP 76 simulation finished, start learning
    0.69
    EP 76 learning finished: 0.10909251363003358, 0.02822325248365841
    77
    EP 77 simulation finished, start learning
    1.07
    EP 77 learning finished: 0.1059481786930958, 0.02651271428571151
    78
    EP 78 simulation finished, start learning
    0.86
    EP 78 learning finished: 0.1323443706175027, 0.025721828525340266
    79
    EP 79 simulation finished, start learning
    1.25
    EP 79 learning finished: 0.09594869117280425, 0.02190429653594149
    80
    EP 80 simulation finished, start learning
    0.88
    EP 80 learning finished: 0.06253913532335353, 0.017818124186723433
    81
    EP 81 simulation finished, start learning
    1.0
    EP 81 learning finished: 0.09108687876697452, 0.024025652029065522
    82
    EP 82 simulation finished, start learning
    0.59
    EP 82 learning finished: 0.12346894653052765, 0.02509702884726881
    83
    EP 83 simulation finished, start learning
    0.66
    EP 83 learning finished: 0.10996819481909981, 0.02687701712081151
    84
    EP 84 simulation finished, start learning
    1.01
    EP 84 learning finished: 0.09777630066722676, 0.02843452790387034
    85
    EP 85 simulation finished, start learning
    0.72
    EP 85 learning finished: 0.0619021922681906, 0.023452312517923513
    86
    EP 86 simulation finished, start learning
    1.04
    EP 86 learning finished: 0.08384098050183475, 0.022126090853683986
    87
    EP 87 simulation finished, start learning
    0.85
    EP 87 learning finished: 0.06885466429455445, 0.02214605394777719
    88
    EP 88 simulation finished, start learning
    1.14
    EP 88 learning finished: 0.10079642816735745, 0.02158085772379839
    89
    EP 89 simulation finished, start learning
    1.11
    EP 89 learning finished: 0.12995917778128163, 0.02262265557285284
    90
    EP 90 simulation finished, start learning
    0.58
    EP 90 learning finished: 0.09421375872840976, 0.02119175921760215
    91
    EP 91 simulation finished, start learning
    1.12
    EP 91 learning finished: 0.06669707995266494, 0.020553615861828675
    92
    EP 92 simulation finished, start learning
    1.48
    EP 92 learning finished: 0.08744774239115971, 0.018425823232936456
    93
    EP 93 simulation finished, start learning
    1.07
    EP 93 learning finished: 0.06438157247054255, 0.016811988419939194
    94
    EP 94 simulation finished, start learning
    1.21
    EP 94 learning finished: 0.05090789318927236, 0.016015209554414607
    95
    EP 95 simulation finished, start learning
    1.01
    EP 95 learning finished: 0.05013010670282014, 0.01630023341330787
    96
    EP 96 simulation finished, start learning
    1.08
    EP 96 learning finished: 0.04930180143034946, 0.01588234723734274
    97
    EP 97 simulation finished, start learning
    1.14
    EP 97 learning finished: 0.062216302122664935, 0.015932069002160237
    98
    EP 98 simulation finished, start learning
    1.05
    EP 98 learning finished: 0.07777799091676219, 0.015508768303579362
    99
    EP 99 simulation finished, start learning
    0.83
    EP 99 learning finished: 0.10060702695065693, 0.01775096891054912
    100
    EP 100 simulation finished, start learning
    0.88
    EP 100 learning finished: 0.12903430267226224, 0.02005621180591958
    101
    EP 101 simulation finished, start learning
    1.33
    EP 101 learning finished: 0.11851054831293517, 0.020186891203401884
    102
    EP 102 simulation finished, start learning
    1.08
    EP 102 learning finished: 0.08277714512707579, 0.02089646164689995
    103
    EP 103 simulation finished, start learning
    1.01
    EP 103 learning finished: 0.05276816578547905, 0.018789936438606998
    104
    EP 104 simulation finished, start learning
    1.29
    EP 104 learning finished: 0.04981711439439618, 0.01721601528448604
    105
    EP 105 simulation finished, start learning
    1.27
    EP 105 learning finished: 0.08972650584238728, 0.016619584917043788
    106
    EP 106 simulation finished, start learning
    0.51
    EP 106 learning finished: 0.06548462614404293, 0.016940191663620326
    107
    EP 107 simulation finished, start learning
    0.9
    EP 107 learning finished: 0.08490464553846522, 0.016879753807403933
    108
    EP 108 simulation finished, start learning
    1.08
    EP 108 learning finished: 0.08227571288910873, 0.014839640561477574
    109
    EP 109 simulation finished, start learning
    0.8
    EP 109 learning finished: 0.0650827193168051, 0.01823254755335635
    110
    EP 110 simulation finished, start learning
    0.86
    EP 110 learning finished: 0.06353788236583346, 0.01663496792937247
    111
    EP 111 simulation finished, start learning
    1.21
    EP 111 learning finished: 0.056792097778202155, 0.01615992784105881
    112
    EP 112 simulation finished, start learning
    0.72
    EP 112 learning finished: 0.08041189363252776, 0.01885459870284123
    113
    EP 113 simulation finished, start learning
    1.2
    EP 113 learning finished: 0.06437231123579679, 0.015121930198644301
    114
    EP 114 simulation finished, start learning
    1.08
    EP 114 learning finished: 0.07212919600080614, 0.016508829172272643
    115
    EP 115 simulation finished, start learning
    1.13
    EP 115 learning finished: 0.08145616569333392, 0.016645134108116147
    116
    EP 116 simulation finished, start learning
    0.92
    EP 116 learning finished: 0.04539390145832822, 0.014738798548547316
    117
    EP 117 simulation finished, start learning
    1.09
    EP 117 learning finished: 0.051357572754747505, 0.015040361813430339
    118
    EP 118 simulation finished, start learning
    1.07
    EP 118 learning finished: 0.07553572243469242, 0.013485288670265758
    119
    EP 119 simulation finished, start learning
    1.02
    EP 119 learning finished: 0.061798886589763846, 0.01259415482342488
    120
    EP 120 simulation finished, start learning
    0.74
    EP 120 learning finished: 0.08351270611676043, 0.01353260787711874
    121
    EP 121 simulation finished, start learning
    0.9
    EP 121 learning finished: 0.07774094164721035, 0.015941606815020742
    122
    EP 122 simulation finished, start learning
    0.73
    EP 122 learning finished: 0.07253666993510421, 0.018046380301369513
    123
    EP 123 simulation finished, start learning
    1.06
    EP 123 learning finished: 0.07630742540785618, 0.017929276970969583
    124
    EP 124 simulation finished, start learning
    1.21
    EP 124 learning finished: 0.07485559044454626, 0.017669579778162935
    125
    EP 125 simulation finished, start learning
    0.95
    EP 125 learning finished: 0.04952881562674172, 0.014811744101639768
    126
    EP 126 simulation finished, start learning
    1.04
    EP 126 learning finished: 0.04856228606838009, 0.011971212217152444
    127
    EP 127 simulation finished, start learning
    1.14
    EP 127 learning finished: 0.027837405447597163, 0.012145705842130795
    128
    EP 128 simulation finished, start learning
    1.08
    EP 128 learning finished: 0.054767753915963446, 0.012128747056302784
    129
    EP 129 simulation finished, start learning
    1.27
    EP 129 learning finished: 0.05354291068470829, 0.013898330732139155
    130
    EP 130 simulation finished, start learning
    0.95
    EP 130 learning finished: 0.0524524600102918, 0.014655997223863404
    131
    EP 131 simulation finished, start learning
    1.18
    EP 131 learning finished: 0.04848956857836573, 0.013750316144319222
    132
    EP 132 simulation finished, start learning
    0.8
    EP 132 learning finished: 0.0707744606430909, 0.015027571832483443
    133
    EP 133 simulation finished, start learning
    0.9
    EP 133 learning finished: 0.053406611911531954, 0.014412614433197527
    134
    EP 134 simulation finished, start learning
    1.35
    EP 134 learning finished: 0.052267810062859965, 0.013496925152203961
    135
    EP 135 simulation finished, start learning
    0.9
    EP 135 learning finished: 0.034254230929325225, 0.013181605598154112
    136
    EP 136 simulation finished, start learning
    0.61
    EP 136 learning finished: 0.0723452550612187, 0.014061422783150115
    137
    EP 137 simulation finished, start learning
    0.91
    EP 137 learning finished: 0.06831647710270072, 0.014225799524795545
    138
    EP 138 simulation finished, start learning
    1.15
    EP 138 learning finished: 0.0577183023959007, 0.015800604385782575
    139
    EP 139 simulation finished, start learning
    1.38
    EP 139 learning finished: 0.056361705443834786, 0.01589426545222357
    140
    EP 140 simulation finished, start learning
    0.92
    EP 140 learning finished: 0.04790397545370739, 0.013240520046916837
    141
    EP 141 simulation finished, start learning
    0.7
    EP 141 learning finished: 0.04718189591766464, 0.011830996436117264
    142
    EP 142 simulation finished, start learning
    0.86
    EP 142 learning finished: 0.046494615988732946, 0.011840030001005586
    143
    EP 143 simulation finished, start learning
    1.16
    EP 143 learning finished: 0.02956205015620128, 0.01112878666339119
    144
    EP 144 simulation finished, start learning
    0.88
    EP 144 learning finished: 0.027037065110302017, 0.010201038757154737
    145
    EP 145 simulation finished, start learning
    1.02
    EP 145 learning finished: 0.04001832587866794, 0.010034540943746562
    146
    EP 146 simulation finished, start learning
    0.89
    EP 146 learning finished: 0.027174938597102738, 0.007751399041579482
    147
    EP 147 simulation finished, start learning
    0.96
    EP 147 learning finished: 0.026636824759072475, 0.008016277622480833
    148
    EP 148 simulation finished, start learning
    1.08
    EP 148 learning finished: 0.02604040629288118, 0.009085237222974454
    149
    EP 149 simulation finished, start learning
    0.77
    EP 149 learning finished: 0.025538589145428503, 0.009946213172664267
    150
    EP 150 simulation finished, start learning
    1.32
    EP 150 learning finished: 0.036792663140172266, 0.011871138663324898
    151
    EP 151 simulation finished, start learning
    0.92
    EP 151 learning finished: 0.03741358445265597, 0.012044739225777493
    152
    EP 152 simulation finished, start learning
    1.34
    EP 152 learning finished: 0.0609897953964218, 0.013396305179437129
    153
    EP 153 simulation finished, start learning
    0.71
    EP 153 learning finished: 0.05935188468167496, 0.013170407169369763
    154
    EP 154 simulation finished, start learning
    0.59
    EP 154 learning finished: 0.05773571712764536, 0.013715080010192657
    155
    EP 155 simulation finished, start learning
    1.18
    EP 155 learning finished: 0.0433182379549244, 0.013014909313958614
    156
    EP 156 simulation finished, start learning
    0.71
    EP 156 learning finished: 0.05154318134948932, 0.012010341273718077
    157
    EP 157 simulation finished, start learning
    1.03
    EP 157 learning finished: 0.06591241094063749, 0.011950919111120312
    158
    EP 158 simulation finished, start learning
    1.05
    EP 158 learning finished: 0.05406537568716807, 0.010794498816899992
    159
    EP 159 simulation finished, start learning
    0.6
    EP 159 learning finished: 0.07873650202121851, 0.01265988410059797
    160
    EP 160 simulation finished, start learning
    1.11
    EP 160 learning finished: 0.063396313700812, 0.01145980172446058
    161
    EP 161 simulation finished, start learning
    0.79
    EP 161 learning finished: 0.04943601582243495, 0.011069439878159188
    162
    EP 162 simulation finished, start learning
    1.19
    EP 162 learning finished: 0.06167826490182149, 0.01184303157870794
    163
    EP 163 simulation finished, start learning
    0.77
    EP 163 learning finished: 0.060410127110889, 0.010150479820045123
    164
    EP 164 simulation finished, start learning
    0.68
    EP 164 learning finished: 0.05917454326984828, 0.01071440417802471
    165
    EP 165 simulation finished, start learning
    0.94
    EP 165 learning finished: 0.038749988214643594, 0.010383749590483271
    166
    EP 166 simulation finished, start learning
    1.07
    EP 166 learning finished: 0.034838707179799755, 0.009373743791392483
    167
    EP 167 simulation finished, start learning
    1.06
    EP 167 learning finished: 0.0342955705364858, 0.010111451404229137
    168
    EP 168 simulation finished, start learning
    0.98
    EP 168 learning finished: 0.029902353801296133, 0.010096559376222794
    169
    EP 169 simulation finished, start learning
    1.03
    EP 169 learning finished: 0.02973518819361276, 0.009594334351843273
    170
    EP 170 simulation finished, start learning
    0.83
    EP 170 learning finished: 0.044242633213401605, 0.009033477021718035
    171
    EP 171 simulation finished, start learning
    1.16
    EP 171 learning finished: 0.0438044828712163, 0.009282111302140982
    172
    EP 172 simulation finished, start learning
    0.6
    EP 172 learning finished: 0.043388540384718066, 0.010643661456746986
    173
    EP 173 simulation finished, start learning
    1.18
    EP 173 learning finished: 0.04295900720302814, 0.010416273289947486
    174
    EP 174 simulation finished, start learning
    0.69
    EP 174 learning finished: 0.044539103173728556, 0.011393029612002253
    175
    EP 175 simulation finished, start learning
    0.97
    EP 175 learning finished: 0.04373762986489371, 0.010153332626298257
    176
    EP 176 simulation finished, start learning
    1.4
    EP 176 learning finished: 0.026304971713535963, 0.00793927132133667
    177
    EP 177 simulation finished, start learning
    1.25
    EP 177 learning finished: 0.018373925869690666, 0.008253045995602053
    178
    EP 178 simulation finished, start learning
    1.02
    EP 178 learning finished: 0.027994102278130484, 0.006683198202835444
    179
    EP 179 simulation finished, start learning
    1.0
    EP 179 learning finished: 0.027185696163360973, 0.00801199848512646
    180
    EP 180 simulation finished, start learning
    0.8
    EP 180 learning finished: 0.02236058535832619, 0.008420940966480338
    181
    EP 181 simulation finished, start learning
    0.79
    EP 181 learning finished: 0.02214959809728434, 0.008415252405833039
    182
    EP 182 simulation finished, start learning
    0.83
    EP 182 learning finished: 0.02670160517899145, 0.009870295081550233
    183
    EP 183 simulation finished, start learning
    1.01
    EP 183 learning finished: 0.026034009230065713, 0.009206377036889053
    184
    EP 184 simulation finished, start learning
    0.75
    EP 184 learning finished: 0.028159230728602935, 0.009438011895450439
    185
    EP 185 simulation finished, start learning
    0.7
    EP 185 learning finished: 0.033282200260337014, 0.00978119716078589
    186
    EP 186 simulation finished, start learning
    0.87
    EP 186 learning finished: 0.03252406236495897, 0.008777438790087015
    187
    EP 187 simulation finished, start learning
    0.66
    EP 187 learning finished: 0.04283596317856819, 0.010236800366817636
    188
    EP 188 simulation finished, start learning
    1.09
    EP 188 learning finished: 0.04184015214115, 0.010223051289166063
    189
    EP 189 simulation finished, start learning
    1.03
    EP 189 learning finished: 0.041010405135493144, 0.010308329889075492
    190
    EP 190 simulation finished, start learning
    1.38
    EP 190 learning finished: 0.04021453023701049, 0.00968033136075294
    191
    EP 191 simulation finished, start learning
    1.0
    EP 191 learning finished: 0.03299250228781292, 0.008040366245932535
    192
    EP 192 simulation finished, start learning
    0.96
    EP 192 learning finished: 0.0353324576142624, 0.008164332166174808
    193
    EP 193 simulation finished, start learning
    0.86
    EP 193 learning finished: 0.03487468698377505, 0.007065595016880671
    194
    EP 194 simulation finished, start learning
    0.74
    EP 194 learning finished: 0.03443912626395029, 0.00843898339107483
    195
    EP 195 simulation finished, start learning
    0.55
    EP 195 learning finished: 0.033998633793666855, 0.009105936694111263
    196
    EP 196 simulation finished, start learning
    1.22
    EP 196 learning finished: 0.023990842036479165, 0.00821875628609143
    197
    EP 197 simulation finished, start learning
    0.71
    EP 197 learning finished: 0.03102861029469306, 0.009586739412269169
    198
    EP 198 simulation finished, start learning
    1.01
    EP 198 learning finished: 0.030313651811586517, 0.008539020949570224
    199
    EP 199 simulation finished, start learning
    1.18
    EP 199 learning finished: 0.029970224321306915, 0.007772546516498141
    200
    EP 200 simulation finished, start learning
    1.24
    EP 200 learning finished: 0.029455755581469845, 0.0074469681608189675
    201
    EP 201 simulation finished, start learning
    0.79
    EP 201 learning finished: 0.03228858436219145, 0.007914854571897821
    202
    EP 202 simulation finished, start learning
    1.08
    EP 202 learning finished: 0.03201078237785265, 0.0077175042252151885
    203
    EP 203 simulation finished, start learning
    0.98
    EP 203 learning finished: 0.04755133816211027, 0.0075241098998517515
    204
    EP 204 simulation finished, start learning
    1.09
    EP 204 learning finished: 0.06246813740516394, 0.008525331471356293
    205
    EP 205 simulation finished, start learning
    0.9
    EP 205 learning finished: 0.030839941548565397, 0.007289457452781624
    206
    EP 206 simulation finished, start learning
    1.36
    EP 206 learning finished: 0.030595158628822894, 0.0068234955312382745
    207
    EP 207 simulation finished, start learning
    1.05
    EP 207 learning finished: 0.030362599538293678, 0.0071167974606625856
    208
    EP 208 simulation finished, start learning
    1.11
    EP 208 learning finished: 0.02813930442122725, 0.006577018232256665
    209
    EP 209 simulation finished, start learning
    0.87
    EP 209 learning finished: 0.027731600891261365, 0.006211150346604204
    210
    EP 210 simulation finished, start learning
    0.85
    EP 210 learning finished: 0.044730479716488566, 0.007674119870706691
    211
    EP 211 simulation finished, start learning
    1.07
    EP 211 learning finished: 0.0295182905849901, 0.007418544022969622
    212
    EP 212 simulation finished, start learning
    1.21
    EP 212 learning finished: 0.029295716614457123, 0.0068733037948953465
    213
    EP 213 simulation finished, start learning
    0.94
    EP 213 learning finished: 0.0434769012454643, 0.007069659952596146
    214
    EP 214 simulation finished, start learning
    0.97
    EP 214 learning finished: 0.018053820584688296, 0.006534056797137433
    215
    EP 215 simulation finished, start learning
    0.81
    EP 215 learning finished: 0.026939861744776472, 0.006980777577815133
    216
    EP 216 simulation finished, start learning
    1.08
    EP 216 learning finished: 0.026782709807905647, 0.006948620324817656
    217
    EP 217 simulation finished, start learning
    0.76
    EP 217 learning finished: 0.026632254501173902, 0.00784539087419877
    218
    EP 218 simulation finished, start learning
    1.04
    EP 218 learning finished: 0.034683740268338425, 0.007454453280935691
    219
    EP 219 simulation finished, start learning
    0.92
    EP 219 learning finished: 0.04249506546381987, 0.0074734728264821415
    220
    EP 220 simulation finished, start learning
    0.99
    EP 220 learning finished: 0.03329171030117628, 0.0075954786607862305
    221
    EP 221 simulation finished, start learning
    1.44
    EP 221 learning finished: 0.026575812197147997, 0.006374775454199021
    222
    EP 222 simulation finished, start learning
    0.95
    EP 222 learning finished: 0.0285697152190707, 0.006812465615743873
    223
    EP 223 simulation finished, start learning
    0.92
    EP 223 learning finished: 0.024054171624657172, 0.006441589082852223
    224
    EP 224 simulation finished, start learning
    0.71
    EP 224 learning finished: 0.031639710543803545, 0.007438525074406819
    225
    EP 225 simulation finished, start learning
    1.0
    EP 225 learning finished: 0.032854116930533195, 0.008436731695245114
    226
    EP 226 simulation finished, start learning
    1.01
    EP 226 learning finished: 0.02577723078917682, 0.00823727361178847
    227
    EP 227 simulation finished, start learning
    0.99
    EP 227 learning finished: 0.025629993135138474, 0.007965849466790784
    228
    EP 228 simulation finished, start learning
    1.21
    EP 228 learning finished: 0.01615032521727855, 0.006428400303530413
    229
    EP 229 simulation finished, start learning
    0.82
    EP 229 learning finished: 0.025435498280506286, 0.006440810675600421
    230
    EP 230 simulation finished, start learning
    1.21
    EP 230 learning finished: 0.0252835632011722, 0.006828939810352912
    231
    EP 231 simulation finished, start learning
    1.01
    EP 231 learning finished: 0.025156605232151907, 0.006885079931025648
    232
    EP 232 simulation finished, start learning
    1.03
    EP 232 learning finished: 0.025008791766320204, 0.007455222549440602
    233
    EP 233 simulation finished, start learning
    1.01
    EP 233 learning finished: 0.027595318960411674, 0.007982809512142097
    234
    EP 234 simulation finished, start learning
    1.45
    EP 234 learning finished: 0.029387867296996717, 0.007036179050358166
    235
    EP 235 simulation finished, start learning
    0.88
    EP 235 learning finished: 0.027095498512885285, 0.007502625543816942
    236
    EP 236 simulation finished, start learning
    1.09
    EP 236 learning finished: 0.028791113301746663, 0.0076766496836288625
    237
    EP 237 simulation finished, start learning
    0.91
    EP 237 learning finished: 0.03675096583377102, 0.0065282782150027155
    238
    EP 238 simulation finished, start learning
    1.2
    EP 238 learning finished: 0.03642870993787628, 0.006711360727698509
    239
    EP 239 simulation finished, start learning
    1.14
    EP 239 learning finished: 0.015572282840926377, 0.006249330612263106
    240
    EP 240 simulation finished, start learning
    0.94
    EP 240 learning finished: 0.014591171242336998, 0.005892145062586269
    241
    EP 241 simulation finished, start learning
    0.78
    EP 241 learning finished: 0.02431851509472427, 0.006565262094750731
    242
    EP 242 simulation finished, start learning
    1.14
    EP 242 learning finished: 0.03540211056031506, 0.00648420078646113
    243
    EP 243 simulation finished, start learning
    0.88
    EP 243 learning finished: 0.03424858541836073, 0.007191620477513185
    244
    EP 244 simulation finished, start learning
    1.36
    EP 244 learning finished: 0.033090408813432504, 0.007752962805616832
    245
    EP 245 simulation finished, start learning
    1.01
    EP 245 learning finished: 0.020148545377042026, 0.006337189104195845
    246
    EP 246 simulation finished, start learning
    0.51
    EP 246 learning finished: 0.019631475792516007, 0.007200158023872203
    247
    EP 247 simulation finished, start learning
    1.03
    EP 247 learning finished: 0.02380555382644145, 0.007317808141828222
    248
    EP 248 simulation finished, start learning
    0.79
    EP 248 learning finished: 0.027997866342173577, 0.007820148062910703
    249
    EP 249 simulation finished, start learning
    0.29
    EP 249 learning finished: 0.035279928337202504, 0.009307949839811383
    250
    EP 250 simulation finished, start learning
    0.96
    EP 250 learning finished: 0.0413114361688493, 0.0087856496196996
    251
    EP 251 simulation finished, start learning
    1.25
    EP 251 learning finished: 0.040837660730380754, 0.00802914130600016
    252
    EP 252 simulation finished, start learning
    0.96
    EP 252 learning finished: 0.02698320316058589, 0.0066707130693333515
    253
    EP 253 simulation finished, start learning
    1.27
    EP 253 learning finished: 0.040106379205075815, 0.005722333896982313
    254
    EP 254 simulation finished, start learning
    0.99
    EP 254 learning finished: 0.026487972014845984, 0.005476980983984584
    255
    EP 255 simulation finished, start learning
    0.92
    EP 255 learning finished: 0.018961004428672545, 0.004753634030994203
    256
    EP 256 simulation finished, start learning
    1.14
    EP 256 learning finished: 0.018682316335348226, 0.004905925450406294
    257
    EP 257 simulation finished, start learning
    0.83
    EP 257 learning finished: 0.014114626479578618, 0.005350282466769931
    258
    EP 258 simulation finished, start learning
    1.02
    EP 258 learning finished: 0.01525449516685029, 0.0054856859038711035
    259
    EP 259 simulation finished, start learning
    1.14
    EP 259 learning finished: 0.026084376124336006, 0.005939747705592517
    260
    EP 260 simulation finished, start learning
    1.07
    EP 260 learning finished: 0.025880449314027554, 0.0059336623848225915
    261
    EP 261 simulation finished, start learning
    0.95
    EP 261 learning finished: 0.03847526835337034, 0.00650850754248338
    262
    EP 262 simulation finished, start learning
    0.75
    EP 262 learning finished: 0.038417467672624994, 0.007626858039472021
    263
    EP 263 simulation finished, start learning
    1.19
    EP 263 learning finished: 0.03723980066553967, 0.007325433678080737
    264
    EP 264 simulation finished, start learning
    0.95
    EP 264 learning finished: 0.028405084447651774, 0.007506983754192925
    265
    EP 265 simulation finished, start learning
    0.88
    EP 265 learning finished: 0.026554756500690502, 0.006246645564378733
    266
    EP 266 simulation finished, start learning
    1.08
    EP 266 learning finished: 0.02618354677417223, 0.004904493726968551
    267
    EP 267 simulation finished, start learning
    0.94
    EP 267 learning finished: 0.025820832801823457, 0.004783592178076251
    268
    EP 268 simulation finished, start learning
    1.05
    EP 268 learning finished: 0.019126297167279205, 0.004629409512895977
    269
    EP 269 simulation finished, start learning
    0.9
    EP 269 learning finished: 0.012642304664721493, 0.00533647309543181
    270
    EP 270 simulation finished, start learning
    1.38
    EP 270 learning finished: 0.011710655716664853, 0.004768588925284756
    271
    EP 271 simulation finished, start learning
    0.71
    EP 271 learning finished: 0.012638828813600456, 0.005129403582711679
    272
    EP 272 simulation finished, start learning
    1.22
    EP 272 learning finished: 0.02293016311145557, 0.0057551457705532355
    273
    EP 273 simulation finished, start learning
    1.25
    EP 273 learning finished: 0.015946604742503645, 0.005080818810087935
    274
    EP 274 simulation finished, start learning
    1.33
    EP 274 learning finished: 0.015861369064450237, 0.005176510014016207
    275
    EP 275 simulation finished, start learning
    1.32
    EP 275 learning finished: 0.015774341618739918, 0.00548608812213559
    276
    EP 276 simulation finished, start learning
    0.95
    EP 276 learning finished: 0.012444404673600307, 0.004776397401660823
    277
    EP 277 simulation finished, start learning
    0.75
    EP 277 learning finished: 0.024759051279557465, 0.00525431301679824
    278
    EP 278 simulation finished, start learning
    1.05
    EP 278 learning finished: 0.02458217102734661, 0.0057607433929614295
    279
    EP 279 simulation finished, start learning
    1.18
    EP 279 learning finished: 0.02252358303330715, 0.005479969911382384
    280
    EP 280 simulation finished, start learning
    1.05
    EP 280 learning finished: 0.018130420184237828, 0.006265490846104945
    281
    EP 281 simulation finished, start learning
    1.16
    EP 281 learning finished: 0.026886036707871597, 0.005842187178619833
    282
    EP 282 simulation finished, start learning
    1.08
    EP 282 learning finished: 0.028781811059961804, 0.006374541856684263
    283
    EP 283 simulation finished, start learning
    1.25
    EP 283 learning finished: 0.028013609149249774, 0.006551263872174834
    284
    EP 284 simulation finished, start learning
    1.18
    EP 284 learning finished: 0.027297187820415125, 0.005336642153682892
    285
    EP 285 simulation finished, start learning
    0.75
    EP 285 learning finished: 0.033144299831836754, 0.00628385169663968
    286
    EP 286 simulation finished, start learning
    0.88
    EP 286 learning finished: 0.019409269184888167, 0.0062331353577137465
    287
    EP 287 simulation finished, start learning
    0.99
    EP 287 learning finished: 0.01524604637034166, 0.005941891324615665
    288
    EP 288 simulation finished, start learning
    0.98
    EP 288 learning finished: 0.021268393244956663, 0.006570736330555864
    289
    EP 289 simulation finished, start learning
    1.01
    EP 289 learning finished: 0.03120620874713076, 0.006932524627654166
    290
    EP 290 simulation finished, start learning
    1.09
    EP 290 learning finished: 0.020344668364697505, 0.005899696261408766
    291
    EP 291 simulation finished, start learning
    0.8
    EP 291 learning finished: 0.019984923586762626, 0.007145163722209316
    292
    EP 292 simulation finished, start learning
    0.86
    EP 292 learning finished: 0.016723456512156787, 0.0069787595302403665
    293
    EP 293 simulation finished, start learning
    0.78
    EP 293 learning finished: 0.013730273635181867, 0.0058818886452097
    294
    EP 294 simulation finished, start learning
    0.79
    EP 294 learning finished: 0.018126979471479565, 0.006959971489203558
    295
    EP 295 simulation finished, start learning
    0.95
    EP 295 learning finished: 0.04329621005973028, 0.006223575916398133
    296
    EP 296 simulation finished, start learning
    0.93
    EP 296 learning finished: 0.04178393627718879, 0.006615172367054022
    297
    EP 297 simulation finished, start learning
    1.03
    EP 297 learning finished: 0.040330818317492856, 0.006458298517637899
    298
    EP 298 simulation finished, start learning
    1.23
    EP 298 learning finished: 0.02355359849590144, 0.005651195490026889
    299
    EP 299 simulation finished, start learning
    0.94
    EP 299 learning finished: 0.02321747235928706, 0.005289117608673568
    300
    EP 300 simulation finished, start learning
    1.03
    EP 300 learning finished: 0.02577122293524603, 0.004981530035403832
    301
    EP 301 simulation finished, start learning
    1.25
    EP 301 learning finished: 0.024659165700755104, 0.005426383965821888
    302
    EP 302 simulation finished, start learning
    0.92
    EP 302 learning finished: 0.03405408290041456, 0.006206474768800062
    303
    EP 303 simulation finished, start learning
    0.78
    EP 303 learning finished: 0.02404262675488411, 0.0059285193021862345
    304
    EP 304 simulation finished, start learning
    1.44
    EP 304 learning finished: 0.023763824652824894, 0.005241401556015254
    305
    EP 305 simulation finished, start learning
    0.61
    EP 305 learning finished: 0.01568043462401114, 0.004714122162766045
    306
    EP 306 simulation finished, start learning
    0.79
    EP 306 learning finished: 0.01766103637756622, 0.005071457975598846
    307
    EP 307 simulation finished, start learning
    1.08
    EP 307 learning finished: 0.02651523616388385, 0.005853894147048005
    308
    EP 308 simulation finished, start learning
    1.01
    EP 308 learning finished: 0.026207878069316237, 0.006322791027835055
    309
    EP 309 simulation finished, start learning
    1.14
    EP 309 learning finished: 0.025923583873130873, 0.0062195912880504255
    310
    EP 310 simulation finished, start learning
    1.04
    EP 310 learning finished: 0.025635730463694273, 0.005675038047696162
    311
    EP 311 simulation finished, start learning
    0.92
    EP 311 learning finished: 0.01444414596882826, 0.004421805850913658
    312
    EP 312 simulation finished, start learning
    1.07
    EP 312 learning finished: 0.019454941433942885, 0.0044060127874743
    313
    EP 313 simulation finished, start learning
    0.98
    EP 313 learning finished: 0.022128414073069536, 0.004410594631207463
    314
    EP 314 simulation finished, start learning
    1.18
    EP 314 learning finished: 0.014629771042518058, 0.004787401984378746
    315
    EP 315 simulation finished, start learning
    1.17
    EP 315 learning finished: 0.022221097513859034, 0.00580612791905581
    316
    EP 316 simulation finished, start learning
    0.92
    EP 316 learning finished: 0.02207519293469873, 0.005705372665630394
    317
    EP 317 simulation finished, start learning
    1.18
    EP 317 learning finished: 0.02192610080528179, 0.0056241808492858685
    318
    EP 318 simulation finished, start learning
    0.94
    EP 318 learning finished: 0.014074135357247997, 0.004804969535041418
    319
    EP 319 simulation finished, start learning
    1.18
    EP 319 learning finished: 0.020674338921930158, 0.004406820550963392
    320
    EP 320 simulation finished, start learning
    0.9
    EP 320 learning finished: 0.013981151292842497, 0.004075448755106209
    321
    EP 321 simulation finished, start learning
    1.13
    EP 321 learning finished: 0.013918479828535446, 0.003932419622847766
    322
    EP 322 simulation finished, start learning
    0.73
    EP 322 learning finished: 0.02762161922212858, 0.004150213703287137
    323
    EP 323 simulation finished, start learning
    1.16
    EP 323 learning finished: 0.013693820474173002, 0.0037438571580366564
    324
    EP 324 simulation finished, start learning
    1.14
    EP 324 learning finished: 0.027183839343356553, 0.004889456964110453
    325
    EP 325 simulation finished, start learning
    0.83
    EP 325 learning finished: 0.026890885945434206, 0.00560231312686003
    326
    EP 326 simulation finished, start learning
    1.08
    EP 326 learning finished: 0.022773294966705215, 0.005605259833540086
    327
    EP 327 simulation finished, start learning
    0.7
    EP 327 learning finished: 0.02648537683447305, 0.006228781976862769
    328
    EP 328 simulation finished, start learning
    0.62
    EP 328 learning finished: 0.02176625982794711, 0.0057323653164549696
    329
    EP 329 simulation finished, start learning
    0.94
    EP 329 learning finished: 0.013071836511814539, 0.004677644434178313
    330
    EP 330 simulation finished, start learning
    0.77
    EP 330 learning finished: 0.013296685204005954, 0.005010063262747458
    331
    EP 331 simulation finished, start learning
    0.78
    EP 331 learning finished: 0.017557993195664423, 0.0053695926987551035
    332
    EP 332 simulation finished, start learning
    1.36
    EP 332 learning finished: 0.01734925490322503, 0.004931108000405531
    333
    EP 333 simulation finished, start learning
    0.92
    EP 333 learning finished: 0.01715574464352798, 0.005293484041255448
    334
    EP 334 simulation finished, start learning
    0.75
    EP 334 learning finished: 0.01785278352013364, 0.005044709886889848
    335
    EP 335 simulation finished, start learning
    0.94
    EP 335 learning finished: 0.014013570440461365, 0.00421643641855534
    336
    EP 336 simulation finished, start learning
    0.9
    EP 336 learning finished: 0.015026410384006539, 0.0047271363575918796
    337
    EP 337 simulation finished, start learning
    1.25
    EP 337 learning finished: 0.014792512651268908, 0.004433903392389957
    338
    EP 338 simulation finished, start learning
    0.85
    EP 338 learning finished: 0.014591386521788152, 0.004182553183386173
    339
    EP 339 simulation finished, start learning
    1.16
    EP 339 learning finished: 0.014377334868479252, 0.004133672041452124
    340
    EP 340 simulation finished, start learning
    1.18
    EP 340 learning finished: 0.01077026695977068, 0.0037431848122265465
    341
    EP 341 simulation finished, start learning
    0.97
    EP 341 learning finished: 0.013848189106299813, 0.004286596328049064
    342
    EP 342 simulation finished, start learning
    0.73
    EP 342 learning finished: 0.020591098858982804, 0.004596100955820054
    343
    EP 343 simulation finished, start learning
    1.4
    EP 343 learning finished: 0.01360227056231711, 0.004239490205278501
    344
    EP 344 simulation finished, start learning
    1.36
    EP 344 learning finished: 0.013525138326400565, 0.004282817566466368
    345
    EP 345 simulation finished, start learning
    1.18
    EP 345 learning finished: 0.017582894084865686, 0.004368273663906713
    346
    EP 346 simulation finished, start learning
    0.53
    EP 346 learning finished: 0.011570797649440578, 0.004009405636249047
    347
    EP 347 simulation finished, start learning
    1.27
    EP 347 learning finished: 0.024659175907357778, 0.004708440270650122
    348
    EP 348 simulation finished, start learning
    1.09
    EP 348 learning finished: 0.024457682856645846, 0.004460862800635179
    349
    EP 349 simulation finished, start learning
    1.11
    EP 349 learning finished: 0.01745229292953887, 0.004032718046774297
    350
    EP 350 simulation finished, start learning
    0.97
    EP 350 learning finished: 0.017169990515115252, 0.004258410393016998
    351
    EP 351 simulation finished, start learning
    0.66
    EP 351 learning finished: 0.012594766502977528, 0.004031688375314896
    352
    EP 352 simulation finished, start learning
    1.47
    EP 352 learning finished: 0.01254291906679228, 0.004027556997082987
    353
    EP 353 simulation finished, start learning
    1.26
    EP 353 learning finished: 0.015597109522849095, 0.0038767182238395315
    354
    EP 354 simulation finished, start learning
    1.22
    EP 354 learning finished: 0.015426214229574242, 0.0035840057150149664
    355
    EP 355 simulation finished, start learning
    0.73
    EP 355 learning finished: 0.010542585967872456, 0.003329493609583649
    356
    EP 356 simulation finished, start learning
    1.4
    EP 356 learning finished: 0.021011385814519848, 0.003872387984820073
    357
    EP 357 simulation finished, start learning
    1.16
    EP 357 learning finished: 0.020901402686694187, 0.0039074879304729125
    358
    EP 358 simulation finished, start learning
    1.05
    EP 358 learning finished: 0.012193301268689893, 0.0036567344556698127
    359
    EP 359 simulation finished, start learning
    1.2
    EP 359 learning finished: 0.013559846781083351, 0.0036007673506832016
    360
    EP 360 simulation finished, start learning
    1.16
    EP 360 learning finished: 0.013428862286474441, 0.0035337468205549893
    361
    EP 361 simulation finished, start learning
    0.96
    EP 361 learning finished: 0.01945952727144995, 0.0038703813797074937
    362
    EP 362 simulation finished, start learning
    0.77
    EP 362 learning finished: 0.01925288120960822, 0.004472117976847094
    363
    EP 363 simulation finished, start learning
    0.7
    EP 363 learning finished: 0.019042811759969247, 0.005053458501524441
    364
    EP 364 simulation finished, start learning
    1.34
    EP 364 learning finished: 0.02049420907209143, 0.004490979308623479
    365
    EP 365 simulation finished, start learning
    0.59
    EP 365 learning finished: 0.0200933820291187, 0.0045774935924109885
    366
    EP 366 simulation finished, start learning
    0.56
    EP 366 learning finished: 0.02136578804981526, 0.005048632796544482
    367
    EP 367 simulation finished, start learning
    1.16
    EP 367 learning finished: 0.021227861811375037, 0.004470762636697413
    368
    EP 368 simulation finished, start learning
    0.64
    EP 368 learning finished: 0.021106121629718366, 0.0053587260077252175
    369
    EP 369 simulation finished, start learning
    1.01
    EP 369 learning finished: 0.0209878484370023, 0.0051962353525595514
    370
    EP 370 simulation finished, start learning
    0.83
    EP 370 learning finished: 0.014131107885794414, 0.00441870167499972
    371
    EP 371 simulation finished, start learning
    1.31
    EP 371 learning finished: 0.01467433590724987, 0.004474561292184681
    372
    EP 372 simulation finished, start learning
    1.35
    EP 372 learning finished: 0.010765252455477992, 0.00352220060670827
    373
    EP 373 simulation finished, start learning
    1.01
    EP 373 learning finished: 0.012541321423138463, 0.003611428202084127
    374
    EP 374 simulation finished, start learning
    1.09
    EP 374 learning finished: 0.012484813343010437, 0.003490298631221774
    375
    EP 375 simulation finished, start learning
    0.84
    EP 375 learning finished: 0.01242232076778027, 0.0036222685829699055
    376
    EP 376 simulation finished, start learning
    0.98
    EP 376 learning finished: 0.015590471244959048, 0.004090607924496688
    377
    EP 377 simulation finished, start learning
    1.54
    EP 377 learning finished: 0.010314748951445907, 0.003552067014858778
    378
    EP 378 simulation finished, start learning
    0.82
    EP 378 learning finished: 0.013181290015957359, 0.0038453912058639683
    379
    EP 379 simulation finished, start learning
    0.74
    EP 379 learning finished: 0.012995612456361272, 0.003801729291780881
    380
    EP 380 simulation finished, start learning
    0.95
    EP 380 learning finished: 0.022154692756139127, 0.0037761449836426486
    381
    EP 381 simulation finished, start learning
    1.36
    EP 381 learning finished: 0.03397309240169477, 0.004432104548712151
    382
    EP 382 simulation finished, start learning
    0.92
    EP 382 learning finished: 0.03345928821446709, 0.004615668063329505
    383
    EP 383 simulation finished, start learning
    1.01
    EP 383 learning finished: 0.023545172613391507, 0.004794282055103172
    384
    EP 384 simulation finished, start learning
    0.77
    EP 384 learning finished: 0.020854289907390466, 0.005200699602470573
    385
    EP 385 simulation finished, start learning
    0.99
    EP 385 learning finished: 0.020722578719398532, 0.0046725501339069265
    386
    EP 386 simulation finished, start learning
    1.2
    EP 386 learning finished: 0.020593569188549488, 0.004265501874148057
    387
    EP 387 simulation finished, start learning
    1.09
    EP 387 learning finished: 0.020465376974944993, 0.004189260859004805
    388
    EP 388 simulation finished, start learning
    0.41
    EP 388 learning finished: 0.02033845089907782, 0.004190474526240162
    389
    EP 389 simulation finished, start learning
    1.27
    EP 389 learning finished: 0.02021097765466351, 0.004172949381866131
    390
    EP 390 simulation finished, start learning
    0.7
    EP 390 learning finished: 0.02008740786250418, 0.003982535959100288
    391
    EP 391 simulation finished, start learning
    0.74
    EP 391 learning finished: 0.01995978597177861, 0.00396221754055779
    392
    EP 392 simulation finished, start learning
    0.99
    EP 392 learning finished: 0.009933525609343574, 0.003525027851182659
    393
    EP 393 simulation finished, start learning
    0.9
    EP 393 learning finished: 0.019784235723452337, 0.004224972928617218
    394
    EP 394 simulation finished, start learning
    0.97
    EP 394 learning finished: 0.010113395392048752, 0.00418318750665716
    395
    EP 395 simulation finished, start learning
    1.14
    EP 395 learning finished: 0.014121782168056818, 0.0042991030104189126
    396
    EP 396 simulation finished, start learning
    1.05
    EP 396 learning finished: 0.011438056227122573, 0.004184779891913409
    397
    EP 397 simulation finished, start learning
    0.87
    EP 397 learning finished: 0.013807804445744587, 0.0035548407720300546
    398
    EP 398 simulation finished, start learning
    1.16
    EP 398 learning finished: 0.01133187828331117, 0.00345104263197471
    399
    EP 399 simulation finished, start learning
    0.93
    EP 399 learning finished: 0.007903900462209457, 0.0032870899763996486
    400
    EP 400 simulation finished, start learning
    0.73
    EP 400 learning finished: 0.007851928414045695, 0.003096542400935043
    401
    EP 401 simulation finished, start learning
    0.81
    EP 401 learning finished: 0.011211587542443446, 0.0033376167546065316
    402
    EP 402 simulation finished, start learning
    0.94
    EP 402 learning finished: 0.011143578327963866, 0.0038491463173404277
    403
    EP 403 simulation finished, start learning
    0.72
    EP 403 learning finished: 0.010036378118926548, 0.0038986975127459148
    404
    EP 404 simulation finished, start learning
    1.21
    EP 404 learning finished: 0.011280220931965346, 0.004079834535583809
    405
    EP 405 simulation finished, start learning
    1.16
    EP 405 learning finished: 0.022606231307331193, 0.004134796920190167
    406
    EP 406 simulation finished, start learning
    1.1
    EP 406 learning finished: 0.022386214675440108, 0.003904415530661287
    407
    EP 407 simulation finished, start learning
    1.2
    EP 407 learning finished: 0.02217535890951794, 0.0035029925993136993
    408
    EP 408 simulation finished, start learning
    0.95
    EP 408 learning finished: 0.0306981355853031, 0.003738063916205271
    409
    EP 409 simulation finished, start learning
    0.79
    EP 409 learning finished: 0.015739253241058293, 0.0036170555385064626
    410
    EP 410 simulation finished, start learning
    0.88
    EP 410 learning finished: 0.013995386888506545, 0.0035706526960224416
    411
    EP 411 simulation finished, start learning
    1.4
    EP 411 learning finished: 0.01608402685327892, 0.0036401970565902164
    412
    EP 412 simulation finished, start learning
    1.19
    EP 412 learning finished: 0.01593643929849886, 0.0035958892964193523
    413
    EP 413 simulation finished, start learning
    1.01
    EP 413 learning finished: 0.01069295457268904, 0.003971996512548648
    414
    EP 414 simulation finished, start learning
    0.77
    EP 414 learning finished: 0.01064671207060286, 0.004050986779814454
    415
    EP 415 simulation finished, start learning
    1.18
    EP 415 learning finished: 0.010602446144647892, 0.004219201765219428
    416
    EP 416 simulation finished, start learning
    1.14
    EP 416 learning finished: 0.019103482069834588, 0.004457704539737992
    417
    EP 417 simulation finished, start learning
    0.74
    EP 417 learning finished: 0.028440224038973394, 0.0043119746067483925
    418
    EP 418 simulation finished, start learning
    1.03
    EP 418 learning finished: 0.02817739772864436, 0.0043008162632645845
    419
    EP 419 simulation finished, start learning
    1.09
    EP 419 learning finished: 0.020819926915011777, 0.00438734894356378
    420
    EP 420 simulation finished, start learning
    0.99
    EP 420 learning finished: 0.019516322162990707, 0.0038134877943457567
    421
    EP 421 simulation finished, start learning
    0.9
    EP 421 learning finished: 0.015215267639432817, 0.0037631829086589227
    422
    EP 422 simulation finished, start learning
    1.16
    EP 422 learning finished: 0.015018009715075387, 0.003546455319912045
    423
    EP 423 simulation finished, start learning
    1.05
    EP 423 learning finished: 0.0148085877963533, 0.00375510785886116
    424
    EP 424 simulation finished, start learning
    1.03
    EP 424 learning finished: 0.028953665153003905, 0.0041177387896335245
    425
    EP 425 simulation finished, start learning
    1.29
    EP 425 learning finished: 0.028727212203444763, 0.0038487699430859654
    426
    EP 426 simulation finished, start learning
    1.38
    EP 426 learning finished: 0.028504910931604566, 0.003971837525952839
    427
    EP 427 simulation finished, start learning
    1.31
    EP 427 learning finished: 0.018884257521666004, 0.0036766573470028395
    428
    EP 428 simulation finished, start learning
    0.94
    EP 428 learning finished: 0.016693838967714925, 0.0035245723325098635
    429
    EP 429 simulation finished, start learning
    0.6
    EP 429 learning finished: 0.010850352018636045, 0.0038661685869497767
    430
    EP 430 simulation finished, start learning
    0.72
    EP 430 learning finished: 0.010781737799090862, 0.003788221457410458
    431
    EP 431 simulation finished, start learning
    0.6
    EP 431 learning finished: 0.011940782623600654, 0.004167255765475969
    432
    EP 432 simulation finished, start learning
    0.98
    EP 432 learning finished: 0.011832559191191194, 0.003556505724793685
    433
    EP 433 simulation finished, start learning
    0.84
    EP 433 learning finished: 0.010191037758017885, 0.003344660572649205
    434
    EP 434 simulation finished, start learning
    0.83
    EP 434 learning finished: 0.01442573255724433, 0.003513644369983887
    435
    EP 435 simulation finished, start learning
    1.01
    EP 435 learning finished: 0.012683077013278421, 0.003303176946263999
    436
    EP 436 simulation finished, start learning
    0.86
    EP 436 learning finished: 0.018719621837284706, 0.003878038224963562
    437
    EP 437 simulation finished, start learning
    0.76
    EP 437 learning finished: 0.01832456631623458, 0.0037585761680903156
    438
    EP 438 simulation finished, start learning
    0.8
    EP 438 learning finished: 0.012014877542924296, 0.0036634249196190773
    439
    EP 439 simulation finished, start learning
    0.99
    EP 439 learning finished: 0.01715494109704707, 0.0035490676100539964
    440
    EP 440 simulation finished, start learning
    1.12
    EP 440 learning finished: 0.016974384066161363, 0.003346752910282552
    441
    EP 441 simulation finished, start learning
    1.03
    EP 441 learning finished: 0.01363949298314715, 0.0038539287081329754
    442
    EP 442 simulation finished, start learning
    0.75
    EP 442 learning finished: 0.01352539832432953, 0.0036750880586395307
    443
    EP 443 simulation finished, start learning
    0.64
    EP 443 learning finished: 0.013412565586170544, 0.00364097221014038
    444
    EP 444 simulation finished, start learning
    0.9
    EP 444 learning finished: 0.023062612654065617, 0.00470876734827903
    445
    EP 445 simulation finished, start learning
    0.55
    EP 445 learning finished: 0.022464523756905663, 0.00450731307919662
    446
    EP 446 simulation finished, start learning
    1.07
    EP 446 learning finished: 0.02189579857800439, 0.004341791982295642
    447
    EP 447 simulation finished, start learning
    0.77
    EP 447 learning finished: 0.021349843539475444, 0.004185189498061875
    448
    EP 448 simulation finished, start learning
    0.94
    EP 448 learning finished: 0.00930498464921814, 0.003924415233535303
    449
    EP 449 simulation finished, start learning
    0.95
    EP 449 learning finished: 0.010432778320096148, 0.0038589025691469804
    450
    EP 450 simulation finished, start learning
    0.92
    EP 450 learning finished: 0.013690857561388547, 0.0043480008319401985
    451
    EP 451 simulation finished, start learning
    1.31
    EP 451 learning finished: 0.013508819856406618, 0.0043283916784452175
    452
    EP 452 simulation finished, start learning
    1.29
    EP 452 learning finished: 0.011661240452731647, 0.0035862146063373457
    453
    EP 453 simulation finished, start learning
    0.77
    EP 453 learning finished: 0.018419049417711664, 0.004046071120912424
    454
    EP 454 simulation finished, start learning
    1.01
    EP 454 learning finished: 0.01832470222003002, 0.004285209863437684
    455
    EP 455 simulation finished, start learning
    0.86
    EP 455 learning finished: 0.01823109951796531, 0.0046054832553882805
    456
    EP 456 simulation finished, start learning
    0.97
    EP 456 learning finished: 0.01814577595999367, 0.004759387797864931
    457
    EP 457 simulation finished, start learning
    0.64
    EP 457 learning finished: 0.019319119615856195, 0.004781451127996734
    458
    EP 458 simulation finished, start learning
    0.98
    EP 458 learning finished: 0.026925117878126237, 0.004361096065442105
    459
    EP 459 simulation finished, start learning
    0.99
    EP 459 learning finished: 0.02672886513356565, 0.004620632049607377
    460
    EP 460 simulation finished, start learning
    0.64
    EP 460 learning finished: 0.02652757378494419, 0.004387320650158608
    461
    EP 461 simulation finished, start learning
    1.14
    EP 461 learning finished: 0.013835199611863302, 0.0036716093711417694
    462
    EP 462 simulation finished, start learning
    1.07
    EP 462 learning finished: 0.013628105188473683, 0.003408322746530912
    463
    EP 463 simulation finished, start learning
    0.93
    EP 463 learning finished: 0.019085694637260175, 0.0035427088298476406
    464
    EP 464 simulation finished, start learning
    0.84
    EP 464 learning finished: 0.026399773621628597, 0.004230220827349529
    465
    EP 465 simulation finished, start learning
    0.86
    EP 465 learning finished: 0.02607016986905286, 0.004155855420788987
    466
    EP 466 simulation finished, start learning
    1.03
    EP 466 learning finished: 0.02573647772526133, 0.004220806812650163
    467
    EP 467 simulation finished, start learning
    0.81
    EP 467 learning finished: 0.017255070269427186, 0.003976971281142105
    468
    EP 468 simulation finished, start learning
    0.73
    EP 468 learning finished: 0.011360850405756295, 0.003341241747776485
    469
    EP 469 simulation finished, start learning
    1.2
    EP 469 learning finished: 0.01123817536531746, 0.0033453577106966927
    470
    EP 470 simulation finished, start learning
    0.66
    EP 470 learning finished: 0.017489867223171363, 0.0037346693268129477
    471
    EP 471 simulation finished, start learning
    0.96
    EP 471 learning finished: 0.017648032552743942, 0.0037099506688213456
    472
    EP 472 simulation finished, start learning
    0.75
    EP 472 learning finished: 0.01754425153933692, 0.00397848709905731
    473
    EP 473 simulation finished, start learning
    1.02
    EP 473 learning finished: 0.01744333911187068, 0.0042450794947119855
    474
    EP 474 simulation finished, start learning
    0.95
    EP 474 learning finished: 0.018186691870990757, 0.004389894768898242
    475
    EP 475 simulation finished, start learning
    1.1
    EP 475 learning finished: 0.01804015313780205, 0.003904881914111938
    476
    EP 476 simulation finished, start learning
    0.81
    EP 476 learning finished: 0.017900065131065013, 0.0034929043296624135
    477
    EP 477 simulation finished, start learning
    0.96
    EP 477 learning finished: 0.017180354634041883, 0.0031158558452382135
    478
    EP 478 simulation finished, start learning
    0.79
    EP 478 learning finished: 0.017085526406508844, 0.0023824306172876147
    479
    EP 479 simulation finished, start learning
    0.94
    EP 479 learning finished: 0.008651019029478957, 0.0026530475568774144
    480
    EP 480 simulation finished, start learning
    0.66
    EP 480 learning finished: 0.008635903252503074, 0.0025897245896366917
    481
    EP 481 simulation finished, start learning
    0.96
    EP 481 learning finished: 0.008617934806905314, 0.0029269823491676478
    482
    EP 482 simulation finished, start learning
    1.1
    EP 482 learning finished: 0.008723160269026309, 0.0029319478838010365
    483
    EP 483 simulation finished, start learning
    0.68
    EP 483 learning finished: 0.008968490509307259, 0.0028249000391743218
    484
    EP 484 simulation finished, start learning
    0.88
    EP 484 learning finished: 0.01768998432663782, 0.00346648354950109
    485
    EP 485 simulation finished, start learning
    0.92
    EP 485 learning finished: 0.0262738105626088, 0.003350430650840036
    486
    EP 486 simulation finished, start learning
    0.95
    EP 486 learning finished: 0.02595357170542556, 0.003639189623329722
    487
    EP 487 simulation finished, start learning
    1.04
    EP 487 learning finished: 0.01713782976469247, 0.003836241257540819
    488
    EP 488 simulation finished, start learning
    0.65
    EP 488 learning finished: 0.010468245780695717, 0.0037776511779860147
    489
    EP 489 simulation finished, start learning
    0.83
    EP 489 learning finished: 0.011398496495910848, 0.0037321251905527222
    490
    EP 490 simulation finished, start learning
    0.94
    EP 490 learning finished: 0.008540647635116638, 0.0035021125311833073
    491
    EP 491 simulation finished, start learning
    0.96
    EP 491 learning finished: 0.008523783478387559, 0.003329733695000746
    492
    EP 492 simulation finished, start learning
    1.01
    EP 492 learning finished: 0.014994179972548594, 0.003130574667543455
    493
    EP 493 simulation finished, start learning
    0.87
    EP 493 learning finished: 0.008486066112985746, 0.003129036094551396
    494
    EP 494 simulation finished, start learning
    0.62
    EP 494 learning finished: 0.016600483967706303, 0.0040271165229590206
    495
    EP 495 simulation finished, start learning
    0.93
    EP 495 learning finished: 0.02465922105869023, 0.004023415888967097
    496
    EP 496 simulation finished, start learning
    0.68
    EP 496 learning finished: 0.02445147731380759, 0.004005549345058326
    497
    EP 497 simulation finished, start learning
    0.9
    EP 497 learning finished: 0.01619245831413796, 0.0037411368299125197
    498
    EP 498 simulation finished, start learning
    1.01
    EP 498 learning finished: 0.011449845056394459, 0.0028167511350089686
    499
    EP 499 simulation finished, start learning
    0.83
    EP 499 learning finished: 0.011310414972131255, 0.002327618686482076
    500
    EP 500 simulation finished, start learning
    1.07
    EP 500 learning finished: 0.00746903070203242, 0.0021265666274605244
    501
    EP 501 simulation finished, start learning
    0.68
    EP 501 learning finished: 0.0071042091329580614, 0.0024759073623373817
    502
    EP 502 simulation finished, start learning
    0.87
    EP 502 learning finished: 0.02242460445830563, 0.003332984479617246
    503
    EP 503 simulation finished, start learning
    1.27
    EP 503 learning finished: 0.02216666563638814, 0.00351191482639282
    504
    EP 504 simulation finished, start learning
    1.24
    EP 504 learning finished: 0.015779786169455967, 0.003700759998297526
    505
    EP 505 simulation finished, start learning
    0.85
    EP 505 learning finished: 0.015658800452531096, 0.0037945060840864075
    506
    EP 506 simulation finished, start learning
    0.77
    EP 506 learning finished: 0.013111952834364615, 0.003176447329588522
    507
    EP 507 simulation finished, start learning
    1.21
    EP 507 learning finished: 0.0167850952448112, 0.0034347836258838554
    508
    EP 508 simulation finished, start learning
    1.19
    EP 508 learning finished: 0.016704286222738007, 0.003172421642873853
    509
    EP 509 simulation finished, start learning
    0.92
    EP 509 learning finished: 0.016626711028479457, 0.0031247464085715562
    510
    EP 510 simulation finished, start learning
    0.7
    EP 510 learning finished: 0.01655179722783462, 0.0032091425520499733
    511
    EP 511 simulation finished, start learning
    1.04
    EP 511 learning finished: 0.016019469165169387, 0.0033737347339021784
    512
    EP 512 simulation finished, start learning
    0.85
    EP 512 learning finished: 0.015927839288445433, 0.003222857165044106
    513
    EP 513 simulation finished, start learning
    0.9
    EP 513 learning finished: 0.008216960797347683, 0.0029317613527466355
    514
    EP 514 simulation finished, start learning
    0.94
    EP 514 learning finished: 0.015426300252776137, 0.0029553613601635276
    515
    EP 515 simulation finished, start learning
    1.19
    EP 515 learning finished: 0.012424973997013922, 0.0025335639183121927
    516
    EP 516 simulation finished, start learning
    0.9
    EP 516 learning finished: 0.012280702007370081, 0.002699702721135011
    517
    EP 517 simulation finished, start learning
    1.15
    EP 517 learning finished: 0.01777902913780416, 0.0029347657437893415
    518
    EP 518 simulation finished, start learning
    1.08
    EP 518 learning finished: 0.014072186857022828, 0.0025030166217084315
    519
    EP 519 simulation finished, start learning
    0.92
    EP 519 learning finished: 0.013946644350545156, 0.0028131389091702597
    520
    EP 520 simulation finished, start learning
    1.25
    EP 520 learning finished: 0.013816246779603247, 0.002764979461743866
    521
    EP 521 simulation finished, start learning
    1.16
    EP 521 learning finished: 0.008159620674381696, 0.0024543810314853337
    522
    EP 522 simulation finished, start learning
    1.14
    EP 522 learning finished: 0.009122575539162625, 0.0025037615672885982
    523
    EP 523 simulation finished, start learning
    0.94
    EP 523 learning finished: 0.00812887344385893, 0.0022597724083191287
    524
    EP 524 simulation finished, start learning
    1.03
    EP 524 learning finished: 0.016201038093286657, 0.0024358366053574623
    525
    EP 525 simulation finished, start learning
    1.1
    EP 525 learning finished: 0.024161804487660365, 0.0028338957083384325
    526
    EP 526 simulation finished, start learning
    1.2
    EP 526 learning finished: 0.023992312152079975, 0.002992233236216016
    527
    EP 527 simulation finished, start learning
    0.8
    EP 527 learning finished: 0.023825391492588333, 0.0033538180033126155
    528
    EP 528 simulation finished, start learning
    1.1
    EP 528 learning finished: 0.02366367962237148, 0.0034191324631078143
    529
    EP 529 simulation finished, start learning
    1.18
    EP 529 learning finished: 0.01568766020381651, 0.0029587674241546777
    530
    EP 530 simulation finished, start learning
    0.86
    EP 530 learning finished: 0.015617399853893321, 0.003251131216807021
    531
    EP 531 simulation finished, start learning
    0.81
    EP 531 learning finished: 0.008747674343913658, 0.0030999267828197632
    532
    EP 532 simulation finished, start learning
    0.98
    EP 532 learning finished: 0.008453094310182152, 0.0029746166648060013
    533
    EP 533 simulation finished, start learning
    0.88
    EP 533 learning finished: 0.00877043562241564, 0.0029595691066404417
    534
    EP 534 simulation finished, start learning
    1.1
    EP 534 learning finished: 0.008689275167869326, 0.00276212051582371
    535
    EP 535 simulation finished, start learning
    0.81
    EP 535 learning finished: 0.022180076789836356, 0.0030688598190074454
    536
    EP 536 simulation finished, start learning
    0.94
    EP 536 learning finished: 0.021935175527656448, 0.0028661594785182945
    537
    EP 537 simulation finished, start learning
    1.21
    EP 537 learning finished: 0.02168906961171846, 0.003311733495935658
    538
    EP 538 simulation finished, start learning
    1.2
    EP 538 learning finished: 0.014333094533174773, 0.0031460640833010922
    539
    EP 539 simulation finished, start learning
    0.92
    EP 539 learning finished: 0.00962894355535937, 0.0028735672683073282
    540
    EP 540 simulation finished, start learning
    0.98
    EP 540 learning finished: 0.008792297100630009, 0.002894212844249941
    541
    EP 541 simulation finished, start learning
    0.88
    EP 541 learning finished: 0.01579835565295351, 0.0031206830705603136
    542
    EP 542 simulation finished, start learning
    0.88
    EP 542 learning finished: 0.02353134468638851, 0.0038951396285766826
    543
    EP 543 simulation finished, start learning
    1.16
    EP 543 learning finished: 0.023339622988564734, 0.00391775658269386
    544
    EP 544 simulation finished, start learning
    0.9
    EP 544 learning finished: 0.02314679972482514, 0.003835834543419306
    545
    EP 545 simulation finished, start learning
    1.08
    EP 545 learning finished: 0.010090538638385382, 0.003448279296831884
    546
    EP 546 simulation finished, start learning
    0.9
    EP 546 learning finished: 0.02290452460955006, 0.003121965523167486
    547
    EP 547 simulation finished, start learning
    0.51
    EP 547 learning finished: 0.02275236804347669, 0.0030608067060966665
    548
    EP 548 simulation finished, start learning
    0.64
    EP 548 learning finished: 0.022602237721105922, 0.0037411224135228127
    549
    EP 549 simulation finished, start learning
    0.88
    EP 549 learning finished: 0.015217218954194589, 0.0035358001947082343
    550
    EP 550 simulation finished, start learning
    1.16
    EP 550 learning finished: 0.015134210244006407, 0.0029940420213169366
    551
    EP 551 simulation finished, start learning
    1.23
    EP 551 learning finished: 0.015053470696368976, 0.0029444984572413735
    552
    EP 552 simulation finished, start learning
    0.98
    EP 552 learning finished: 0.01179795954426055, 0.0023366534112305715
    553
    EP 553 simulation finished, start learning
    0.88
    EP 553 learning finished: 0.012457647861221943, 0.0023280942720725617
    554
    EP 554 simulation finished, start learning
    0.84
    EP 554 learning finished: 0.013080538320235036, 0.0024871777977512055
    555
    EP 555 simulation finished, start learning
    0.97
    EP 555 learning finished: 0.00808273118646774, 0.002579727758825201
    556
    EP 556 simulation finished, start learning
    0.73
    EP 556 learning finished: 0.009167771916262168, 0.002762791702720993
    557
    EP 557 simulation finished, start learning
    0.96
    EP 557 learning finished: 0.011920666363548271, 0.0030106006424398396
    558
    EP 558 simulation finished, start learning
    0.98
    EP 558 learning finished: 0.011751374959708016, 0.0028620949313970245
    559
    EP 559 simulation finished, start learning
    1.21
    EP 559 learning finished: 0.011588782401920739, 0.002548532720046681
    560
    EP 560 simulation finished, start learning
    0.83
    EP 560 learning finished: 0.014991303106949294, 0.003034873688317334
    561
    EP 561 simulation finished, start learning
    0.96
    EP 561 learning finished: 0.01492700120752155, 0.0033264733660624754
    562
    EP 562 simulation finished, start learning
    1.03
    EP 562 learning finished: 0.014863135906734382, 0.0032780329515633144
    563
    EP 563 simulation finished, start learning
    1.03
    EP 563 learning finished: 0.014799279617234262, 0.0031763858502565356
    564
    EP 564 simulation finished, start learning
    1.03
    EP 564 learning finished: 0.014620677266097726, 0.0025721823260233395
    565
    EP 565 simulation finished, start learning
    0.91
    EP 565 learning finished: 0.005312544084103976, 0.002048621380556448
    566
    EP 566 simulation finished, start learning
    0.81
    EP 566 learning finished: 0.00702135478373922, 0.0023879299261738984
    567
    EP 567 simulation finished, start learning
    1.02
    EP 567 learning finished: 0.007174871446412823, 0.0024751496834709286
    568
    EP 568 simulation finished, start learning
    1.23
    EP 568 learning finished: 0.006991456957413433, 0.0023066503860873777
    569
    EP 569 simulation finished, start learning
    0.86
    EP 569 learning finished: 0.006954070052267447, 0.0021690513346308005
    570
    EP 570 simulation finished, start learning
    0.95
    EP 570 learning finished: 0.0073944923355924175, 0.002203381379635466
    571
    EP 571 simulation finished, start learning
    1.1
    EP 571 learning finished: 0.013794664516167288, 0.002567944040167504
    572
    EP 572 simulation finished, start learning
    1.09
    EP 572 learning finished: 0.01369717591413444, 0.002909420499365399
    573
    EP 573 simulation finished, start learning
    0.9
    EP 573 learning finished: 0.014693288866049148, 0.0031931402035846185
    574
    EP 574 simulation finished, start learning
    0.71
    EP 574 learning finished: 0.0073249525061021, 0.003039347321173974
    575
    EP 575 simulation finished, start learning
    0.79
    EP 575 learning finished: 0.01460552257435932, 0.0027395066950006847
    576
    EP 576 simulation finished, start learning
    0.92
    EP 576 learning finished: 0.014542596128694907, 0.00279175009270205
    577
    EP 577 simulation finished, start learning
    1.11
    EP 577 learning finished: 0.007248543576465982, 0.0026803637951896067
    578
    EP 578 simulation finished, start learning
    1.18
    EP 578 learning finished: 0.00723458900429752, 0.0024049228727203373
    579
    EP 579 simulation finished, start learning
    0.95
    EP 579 learning finished: 0.01432788332776358, 0.002839907180079676
    580
    EP 580 simulation finished, start learning
    1.18
    EP 580 learning finished: 0.007329760499347415, 0.0026718313501667285
    581
    EP 581 simulation finished, start learning
    0.93
    EP 581 learning finished: 0.013515766070609558, 0.0031060026265541385
    582
    EP 582 simulation finished, start learning
    0.73
    EP 582 learning finished: 0.013424719832659804, 0.0034853777267034036
    583
    EP 583 simulation finished, start learning
    1.18
    EP 583 learning finished: 0.013334995681709039, 0.002814483010180585
    584
    EP 584 simulation finished, start learning
    0.73
    EP 584 learning finished: 0.013242690836697157, 0.0025760783950836696
    585
    EP 585 simulation finished, start learning
    0.81
    EP 585 learning finished: 0.011128659568292676, 0.0020965636719784964
    586
    EP 586 simulation finished, start learning
    1.25
    EP 586 learning finished: 0.006448062084290385, 0.0019238683103644516
    587
    EP 587 simulation finished, start learning
    1.05
    EP 587 learning finished: 0.007327694161343601, 0.0020985996105873372
    588
    EP 588 simulation finished, start learning
    1.43
    EP 588 learning finished: 0.007103714235514147, 0.0024648505715746116
    589
    EP 589 simulation finished, start learning
    0.76
    EP 589 learning finished: 0.009810935688215672, 0.0025967000623840913
    590
    EP 590 simulation finished, start learning
    1.22
    EP 590 learning finished: 0.009710086612639834, 0.0026162101567760304
    591
    EP 591 simulation finished, start learning
    1.18
    EP 591 learning finished: 0.013150803710045622, 0.0028848300321607234
    592
    EP 592 simulation finished, start learning
    1.05
    EP 592 learning finished: 0.01800940660872019, 0.002666663964758037
    593
    EP 593 simulation finished, start learning
    0.67
    EP 593 learning finished: 0.017819395183107822, 0.002992392254699411
    594
    EP 594 simulation finished, start learning
    0.9
    EP 594 learning finished: 0.014405666603439782, 0.0027056197500433463
    595
    EP 595 simulation finished, start learning
    0.92
    EP 595 learning finished: 0.014344938109823735, 0.0029095493703408734
    596
    EP 596 simulation finished, start learning
    1.05
    EP 596 learning finished: 0.014283630165159211, 0.002978780786932901
    597
    EP 597 simulation finished, start learning
    1.59
    EP 597 learning finished: 0.011916022179441654, 0.002558528842650426
    598
    EP 598 simulation finished, start learning
    1.19
    EP 598 learning finished: 0.011921036096278925, 0.002904351252213472
    599
    EP 599 simulation finished, start learning
    0.99
    EP 599 learning finished: 0.0148095893822493, 0.0025883476105342806
    600
    EP 600 simulation finished, start learning
    0.93
    EP 600 learning finished: 0.015375712639031036, 0.0026456366555701605
    601
    EP 601 simulation finished, start learning
    0.88
    EP 601 learning finished: 0.014244826902612306, 0.0028727156157550143
    602
    EP 602 simulation finished, start learning
    0.88
    EP 602 learning finished: 0.014185564229706848, 0.00297228779758843
    603
    EP 603 simulation finished, start learning
    1.01
    EP 603 learning finished: 0.014123987874824495, 0.002744986307676729
    604
    EP 604 simulation finished, start learning
    1.32
    EP 604 learning finished: 0.02107306661938857, 0.0027129884496148726
    605
    EP 605 simulation finished, start learning
    0.84
    EP 605 learning finished: 0.01397510835888438, 0.0024878053791668048
    606
    EP 606 simulation finished, start learning
    1.12
    EP 606 learning finished: 0.02085036714884203, 0.002344716030984316
    607
    EP 607 simulation finished, start learning
    0.75
    EP 607 learning finished: 0.02071901246129615, 0.002566569307406799
    608
    EP 608 simulation finished, start learning
    1.07
    EP 608 learning finished: 0.013742150139492537, 0.002569661267563278
    609
    EP 609 simulation finished, start learning
    1.33
    EP 609 learning finished: 0.008100943548164707, 0.002430417579029863
    610
    EP 610 simulation finished, start learning
    0.9
    EP 610 learning finished: 0.008072403824353813, 0.002362258925604939
    611
    EP 611 simulation finished, start learning
    0.95
    EP 611 learning finished: 0.009050481702733038, 0.0022737599252466573
    612
    EP 612 simulation finished, start learning
    0.95
    EP 612 learning finished: 0.011983602883747224, 0.0024222777975784412
    613
    EP 613 simulation finished, start learning
    0.64
    EP 613 learning finished: 0.011885893031925221, 0.0025447196756727478
    614
    EP 614 simulation finished, start learning
    1.05
    EP 614 learning finished: 0.011789214868997533, 0.0024114869556997875
    615
    EP 615 simulation finished, start learning
    1.12
    EP 615 learning finished: 0.008642066418243388, 0.0022155532126028332
    616
    EP 616 simulation finished, start learning
    0.86
    EP 616 learning finished: 0.0067884597239200595, 0.0021346630409535966
    617
    EP 617 simulation finished, start learning
    1.12
    EP 617 learning finished: 0.01202958958654321, 0.0026108285289022162
    618
    EP 618 simulation finished, start learning
    1.21
    EP 618 learning finished: 0.01141055800792401, 0.0026012130964181146
    619
    EP 619 simulation finished, start learning
    1.1
    EP 619 learning finished: 0.01136773583921169, 0.002749689508872409
    620
    EP 620 simulation finished, start learning
    0.6
    EP 620 learning finished: 0.011324886596593586, 0.002557948094971646
    621
    EP 621 simulation finished, start learning
    0.68
    EP 621 learning finished: 0.020920840247443007, 0.002521436852931311
    622
    EP 622 simulation finished, start learning
    1.28
    EP 622 learning finished: 0.0207620226936025, 0.0024650623711918868
    623
    EP 623 simulation finished, start learning
    1.14
    EP 623 learning finished: 0.01375855720469632, 0.002228616709107331
    624
    EP 624 simulation finished, start learning
    0.81
    EP 624 learning finished: 0.013691640952131046, 0.0025932011971633306
    625
    EP 625 simulation finished, start learning
    0.94
    EP 625 learning finished: 0.0067463858160190515, 0.0021208288225057284
    626
    EP 626 simulation finished, start learning
    0.97
    EP 626 learning finished: 0.006732866055256359, 0.0022549491717379355
    627
    EP 627 simulation finished, start learning
    1.05
    EP 627 learning finished: 0.007807054110616107, 0.002450206456586647
    628
    EP 628 simulation finished, start learning
    1.05
    EP 628 learning finished: 0.007771215890695515, 0.00201619419992536
    629
    EP 629 simulation finished, start learning
    0.79
    EP 629 learning finished: 0.007737271355804154, 0.002099001608215051
    630
    EP 630 simulation finished, start learning
    0.85
    EP 630 learning finished: 0.007697991694001871, 0.0022488416945047746
    631
    EP 631 simulation finished, start learning
    1.2
    EP 631 learning finished: 0.00679701729981641, 0.0024584483598345044
    632
    EP 632 simulation finished, start learning
    1.01
    EP 632 learning finished: 0.0067816716134170285, 0.002542026067324489
    633
    EP 633 simulation finished, start learning
    0.84
    EP 633 learning finished: 0.013333632322260591, 0.0026777410687826297
    634
    EP 634 simulation finished, start learning
    1.57
    EP 634 learning finished: 0.013282447024234756, 0.0024309058224298574
    635
    EP 635 simulation finished, start learning
    1.08
    EP 635 learning finished: 0.013227668635007905, 0.002463090193931358
    636
    EP 636 simulation finished, start learning
    0.7
    EP 636 learning finished: 0.013173242789729311, 0.0023742826283536383
    637
    EP 637 simulation finished, start learning
    0.68
    EP 637 learning finished: 0.008646936567336283, 0.002168848678361701
    638
    EP 638 simulation finished, start learning
    0.57
    EP 638 learning finished: 0.013096840628420203, 0.002842829004062991
    639
    EP 639 simulation finished, start learning
    1.26
    EP 639 learning finished: 0.01184329812558893, 0.0025126597434296655
    640
    EP 640 simulation finished, start learning
    0.92
    EP 640 learning finished: 0.010524255150477835, 0.002725804631342893
    641
    EP 641 simulation finished, start learning
    1.03
    EP 641 learning finished: 0.010601973486664917, 0.002675565506072383
    642
    EP 642 simulation finished, start learning
    0.83
    EP 642 learning finished: 0.01297431453023723, 0.0025064275600455155
    643
    EP 643 simulation finished, start learning
    1.27
    EP 643 learning finished: 0.012919444366271016, 0.0025739352598855014
    644
    EP 644 simulation finished, start learning
    1.36
    EP 644 learning finished: 0.012867175772039552, 0.0023097095539548196
    645
    EP 645 simulation finished, start learning
    0.92
    EP 645 learning finished: 0.012816449320671275, 0.0024954085113443047
    646
    EP 646 simulation finished, start learning
    1.16
    EP 646 learning finished: 0.006775092693379392, 0.002083174776714325
    647
    EP 647 simulation finished, start learning
    1.32
    EP 647 learning finished: 0.0074097514165565315, 0.002194552472747996
    648
    EP 648 simulation finished, start learning
    0.84
    EP 648 learning finished: 0.00737352243546141, 0.002257024827970508
    649
    EP 649 simulation finished, start learning
    0.95
    EP 649 learning finished: 0.006729623720654665, 0.002226310587408034
    650
    EP 650 simulation finished, start learning
    1.27
    EP 650 learning finished: 0.006712299822573264, 0.0021786932786483637
    651
    EP 651 simulation finished, start learning
    1.44
    EP 651 learning finished: 0.013379308915871135, 0.0022137530164412778
    652
    EP 652 simulation finished, start learning
    0.8
    EP 652 learning finished: 0.013315768283596263, 0.002198258730777225
    653
    EP 653 simulation finished, start learning
    1.05
    EP 653 learning finished: 0.006634432146067404, 0.002009069236964034
    654
    EP 654 simulation finished, start learning
    0.88
    EP 654 learning finished: 0.006617686095280906, 0.0019432584768478284
    655
    EP 655 simulation finished, start learning
    1.03
    EP 655 learning finished: 0.008478112085047229, 0.0017873754072644799
    656
    EP 656 simulation finished, start learning
    1.11
    EP 656 learning finished: 0.005612563768667478, 0.001784301897909596
    657
    EP 657 simulation finished, start learning
    1.05
    EP 657 learning finished: 0.00660536312722293, 0.0019717135307034395
    658
    EP 658 simulation finished, start learning
    0.88
    EP 658 learning finished: 0.006589326954941743, 0.0020706428334114207
    659
    EP 659 simulation finished, start learning
    0.8
    EP 659 learning finished: 0.006574140606829815, 0.00242444482843579
    660
    EP 660 simulation finished, start learning
    0.77
    EP 660 learning finished: 0.0065596015791205, 0.002660528283449856
    661
    EP 661 simulation finished, start learning
    1.05
    EP 661 learning finished: 0.013070901493052567, 0.0027131214442278756
    662
    EP 662 simulation finished, start learning
    0.81
    EP 662 learning finished: 0.013008535266756827, 0.0027332241639013988
    663
    EP 663 simulation finished, start learning
    1.27
    EP 663 learning finished: 0.012948777967022984, 0.002429569672862064
    664
    EP 664 simulation finished, start learning
    0.69
    EP 664 learning finished: 0.012885205621105289, 0.00244704369972569
    665
    EP 665 simulation finished, start learning
    1.51
    EP 665 learning finished: 0.007172449962455474, 0.0023235861591601598
    666
    EP 666 simulation finished, start learning
    0.74
    EP 666 learning finished: 0.007139976688987915, 0.002262653382746699
    667
    EP 667 simulation finished, start learning
    0.92
    EP 667 learning finished: 0.006294239119268852, 0.002143183355884648
    668
    EP 668 simulation finished, start learning
    1.12
    EP 668 learning finished: 0.006283457005495041, 0.0019191864979663381
    669
    EP 669 simulation finished, start learning
    1.06
    EP 669 learning finished: 0.005964832378602125, 0.002103973426032524
    670
    EP 670 simulation finished, start learning
    0.73
    EP 670 learning finished: 0.008491130054094853, 0.002658421858632798
    671
    EP 671 simulation finished, start learning
    0.81
    EP 671 learning finished: 0.009186152062385489, 0.0025949086143347705
    672
    EP 672 simulation finished, start learning
    0.9
    EP 672 learning finished: 0.009069945336035445, 0.0026477589812721747
    673
    EP 673 simulation finished, start learning
    0.95
    EP 673 learning finished: 0.008037881830543547, 0.0024650096549903994
    674
    EP 674 simulation finished, start learning
    0.99
    EP 674 learning finished: 0.007972311361081452, 0.0019478591041997284
    675
    EP 675 simulation finished, start learning
    0.98
    EP 675 learning finished: 0.007640561225462861, 0.002182084220014077
    676
    EP 676 simulation finished, start learning
    0.93
    EP 676 learning finished: 0.012780848767371289, 0.0029745294399059624
    677
    EP 677 simulation finished, start learning
    0.67
    EP 677 learning finished: 0.012398547741652166, 0.0028526016453901126
    678
    EP 678 simulation finished, start learning
    0.92
    EP 678 learning finished: 0.01851598076779748, 0.003443546063794281
    679
    EP 679 simulation finished, start learning
    1.1
    EP 679 learning finished: 0.018406154981512657, 0.0035804163890618412
    680
    EP 680 simulation finished, start learning
    1.03
    EP 680 learning finished: 0.010888531126797218, 0.002616886469676463
    681
    EP 681 simulation finished, start learning
    1.03
    EP 681 learning finished: 0.012540917172449095, 0.002765681131264977
    682
    EP 682 simulation finished, start learning
    0.88
    EP 682 learning finished: 0.007625398927581628, 0.002252067049229603
    683
    EP 683 simulation finished, start learning
    1.05
    EP 683 learning finished: 0.01245988892238481, 0.002446759771594878
    684
    EP 684 simulation finished, start learning
    0.9
    EP 684 learning finished: 0.012400836554342654, 0.002739809228879675
    685
    EP 685 simulation finished, start learning
    0.64
    EP 685 learning finished: 0.018065917277316323, 0.003308360736225577
    686
    EP 686 simulation finished, start learning
    0.92
    EP 686 learning finished: 0.016256418231239245, 0.0031362303109726958
    687
    EP 687 simulation finished, start learning
    1.16
    EP 687 learning finished: 0.010757554585905105, 0.0028382338868586376
    688
    EP 688 simulation finished, start learning
    0.79
    EP 688 learning finished: 0.010693920758447106, 0.0025960762135186083
    689
    EP 689 simulation finished, start learning
    0.87
    EP 689 learning finished: 0.006125763959077712, 0.001806356043392368
    690
    EP 690 simulation finished, start learning
    0.95
    EP 690 learning finished: 0.0076192814187354685, 0.0019269303224624806
    691
    EP 691 simulation finished, start learning
    0.84
    EP 691 learning finished: 0.007552735511488606, 0.0018145725312171124
    692
    EP 692 simulation finished, start learning
    0.96
    EP 692 learning finished: 0.007490571716024519, 0.0020956231875169373
    693
    EP 693 simulation finished, start learning
    0.96
    EP 693 learning finished: 0.006769951704171717, 0.002076145240792346
    694
    EP 694 simulation finished, start learning
    0.88
    EP 694 learning finished: 0.006626008798209071, 0.0019233678339861948
    695
    EP 695 simulation finished, start learning
    0.86
    EP 695 learning finished: 0.014428297538804813, 0.0023034548253969417
    696
    EP 696 simulation finished, start learning
    0.75
    EP 696 learning finished: 0.00909077870004893, 0.002133421797951812
    697
    EP 697 simulation finished, start learning
    1.0
    EP 697 learning finished: 0.009035540892635474, 0.002275882246879718
    698
    EP 698 simulation finished, start learning
    1.12
    EP 698 learning finished: 0.008985193159888993, 0.0023214850638327428
    699
    EP 699 simulation finished, start learning
    0.88
    EP 699 learning finished: 0.006727435012860994, 0.002341964068639582
    700
    EP 700 simulation finished, start learning
    1.08
    EP 700 learning finished: 0.006699044592631154, 0.002411618112027031
    701
    EP 701 simulation finished, start learning
    0.85
    EP 701 learning finished: 0.01024248614197898, 0.002490128612835457
    702
    EP 702 simulation finished, start learning
    1.01
    EP 702 learning finished: 0.011900416592325769, 0.0028465097252177024
    703
    EP 703 simulation finished, start learning
    0.94
    EP 703 learning finished: 0.011768843728747758, 0.002354990808265428
    704
    EP 704 simulation finished, start learning
    1.07
    EP 704 learning finished: 0.005917308550425826, 0.0021565021965630343
    705
    EP 705 simulation finished, start learning
    1.05
    EP 705 learning finished: 0.0059055876152172004, 0.0018926402723955654
    706
    EP 706 simulation finished, start learning
    1.24
    EP 706 learning finished: 0.0066231170953905405, 0.0016069352226012745
    707
    EP 707 simulation finished, start learning
    0.64
    EP 707 learning finished: 0.006594395207248027, 0.001740773682673863
    708
    EP 708 simulation finished, start learning
    0.83
    EP 708 learning finished: 0.009839957091170248, 0.00201201543504384
    709
    EP 709 simulation finished, start learning
    0.97
    EP 709 learning finished: 0.006525185827895186, 0.0020537601386912407
    710
    EP 710 simulation finished, start learning
    1.18
    EP 710 learning finished: 0.011727486106315244, 0.0019973209788280016
    711
    EP 711 simulation finished, start learning
    1.03
    EP 711 learning finished: 0.011676919598960112, 0.002065733308959178
    712
    EP 712 simulation finished, start learning
    1.07
    EP 712 learning finished: 0.005819479119513193, 0.0018252508820894991
    713
    EP 713 simulation finished, start learning
    0.94
    EP 713 learning finished: 0.007950316684496528, 0.0019494355944687791
    714
    EP 714 simulation finished, start learning
    0.9
    EP 714 learning finished: 0.007884580002745167, 0.00209469664287662
    715
    EP 715 simulation finished, start learning
    0.97
    EP 715 learning finished: 0.007655762355213369, 0.00198872388034782
    716
    EP 716 simulation finished, start learning
    1.17
    EP 716 learning finished: 0.0076157499303288745, 0.0021263612075958643
    717
    EP 717 simulation finished, start learning
    1.09
    EP 717 learning finished: 0.007577981001764655, 0.0020556834807344826
    718
    EP 718 simulation finished, start learning
    1.1
    EP 718 learning finished: 0.01156472890415472, 0.0020515353466221686
    719
    EP 719 simulation finished, start learning
    1.16
    EP 719 learning finished: 0.008593556549105763, 0.001971862552404502
    720
    EP 720 simulation finished, start learning
    0.96
    EP 720 learning finished: 0.005756277051903513, 0.0017580387417538562
    721
    EP 721 simulation finished, start learning
    1.05
    EP 721 learning finished: 0.005745925071367175, 0.0017780850789921032
    722
    EP 722 simulation finished, start learning
    0.92
    EP 722 learning finished: 0.003599435811774576, 0.0017351336601346967
    723
    EP 723 simulation finished, start learning
    1.21
    EP 723 learning finished: 0.007755449417008009, 0.0018651907824219771
    724
    EP 724 simulation finished, start learning
    0.74
    EP 724 learning finished: 0.007258123476056255, 0.0018059847120020234
    725
    EP 725 simulation finished, start learning
    1.1
    EP 725 learning finished: 0.007712374492481189, 0.001724105641025514
    726
    EP 726 simulation finished, start learning
    0.82
    EP 726 learning finished: 0.00911182892768636, 0.002106693801466053
    727
    EP 727 simulation finished, start learning
    1.34
    EP 727 learning finished: 0.011412701527678593, 0.0021911054412248054
    728
    EP 728 simulation finished, start learning
    1.23
    EP 728 learning finished: 0.011848092046037095, 0.002280417810835933
    729
    EP 729 simulation finished, start learning
    0.63
    EP 729 learning finished: 0.011770433535758684, 0.00249560151153988
    730
    EP 730 simulation finished, start learning
    1.05
    EP 730 learning finished: 0.006909016553660052, 0.002136875520787118
    731
    EP 731 simulation finished, start learning
    1.29
    EP 731 learning finished: 0.006402197494864481, 0.002067632301153588
    732
    EP 732 simulation finished, start learning
    0.69
    EP 732 learning finished: 0.006376573445697531, 0.0022167076015545822
    733
    EP 733 simulation finished, start learning
    0.71
    EP 733 learning finished: 0.00986290519728339, 0.002266519049502966
    734
    EP 734 simulation finished, start learning
    0.91
    EP 734 learning finished: 0.009805684405422177, 0.0021047333208626395
    735
    EP 735 simulation finished, start learning
    1.19
    EP 735 learning finished: 0.009752105284252188, 0.0021893779642013222
    736
    EP 736 simulation finished, start learning
    1.13
    EP 736 learning finished: 0.009697263199171857, 0.001900535891849936
    737
    EP 737 simulation finished, start learning
    0.77
    EP 737 learning finished: 0.007103222943225518, 0.0018009607964358243
    738
    EP 738 simulation finished, start learning
    1.24
    EP 738 learning finished: 0.007089047622677924, 0.0018985492712900037
    739
    EP 739 simulation finished, start learning
    1.12
    EP 739 learning finished: 0.0058145479739351025, 0.0016777971912889378
    740
    EP 740 simulation finished, start learning
    0.77
    EP 740 learning finished: 0.013159818963654146, 0.002369661814146093
    741
    EP 741 simulation finished, start learning
    1.1
    EP 741 learning finished: 0.013054723030098359, 0.002125782968335016
    742
    EP 742 simulation finished, start learning
    1.01
    EP 742 learning finished: 0.012950232956288055, 0.0024121921470205837
    743
    EP 743 simulation finished, start learning
    0.81
    EP 743 learning finished: 0.012851587855986502, 0.002529253192311273
    744
    EP 744 simulation finished, start learning
    1.07
    EP 744 learning finished: 0.011477794833155919, 0.002099844637028281
    745
    EP 745 simulation finished, start learning
    1.22
    EP 745 learning finished: 0.017123830779400162, 0.0025083436280956967
    746
    EP 746 simulation finished, start learning
    1.07
    EP 746 learning finished: 0.011354264782185375, 0.0023429529836114654
    747
    EP 747 simulation finished, start learning
    1.21
    EP 747 learning finished: 0.0169380786210942, 0.0028118840500079706
    748
    EP 748 simulation finished, start learning
    0.75
    EP 748 learning finished: 0.02240898949459602, 0.0031180319982401783
    749
    EP 749 simulation finished, start learning
    1.01
    EP 749 learning finished: 0.016676843790872642, 0.0028598671845518897
    750
    EP 750 simulation finished, start learning
    1.4
    EP 750 learning finished: 0.01112704385118679, 0.0027025539730832493
    751
    EP 751 simulation finished, start learning
    1.12
    EP 751 learning finished: 0.007767614829714553, 0.002018033806140934
    752
    EP 752 simulation finished, start learning
    1.12
    EP 752 learning finished: 0.005946633417517333, 0.0016841256063883647
    753
    EP 753 simulation finished, start learning
    0.9
    EP 753 learning finished: 0.00552892932049831, 0.001598971735277635
    754
    EP 754 simulation finished, start learning
    1.01
    EP 754 learning finished: 0.0051017365278536175, 0.0015098725582139705
    755
    EP 755 simulation finished, start learning
    0.64
    EP 755 learning finished: 0.006106896349882263, 0.0019568484091274096
    756
    EP 756 simulation finished, start learning
    0.88
    EP 756 learning finished: 0.006082728130371251, 0.0017701069235464273
    757
    EP 757 simulation finished, start learning
    0.99
    EP 757 learning finished: 0.0060563513530969, 0.0017145966147104896
    758
    EP 758 simulation finished, start learning
    1.14
    EP 758 learning finished: 0.006034052318177885, 0.0017289585195849888
    759
    EP 759 simulation finished, start learning
    0.75
    EP 759 learning finished: 0.005530833992476225, 0.001483819290724192
    760
    EP 760 simulation finished, start learning
    0.94
    EP 760 learning finished: 0.0074772121196629016, 0.001579889901575357
    761
    EP 761 simulation finished, start learning
    1.15
    EP 761 learning finished: 0.007417358665269269, 0.0017349349079456983
    762
    EP 762 simulation finished, start learning
    1.53
    EP 762 learning finished: 0.005504638972317899, 0.0017247879424632795
    763
    EP 763 simulation finished, start learning
    1.01
    EP 763 learning finished: 0.005472366996314371, 0.0017564570361292553
    764
    EP 764 simulation finished, start learning
    1.27
    EP 764 learning finished: 0.005436322948476402, 0.001672977431644962
    765
    EP 765 simulation finished, start learning
    1.16
    EP 765 learning finished: 0.005517656629530077, 0.0016744880151045444
    766
    EP 766 simulation finished, start learning
    1.08
    EP 766 learning finished: 0.007209725135568945, 0.001862860030040367
    767
    EP 767 simulation finished, start learning
    1.15
    EP 767 learning finished: 0.005495196891972309, 0.0017357201666165917
    768
    EP 768 simulation finished, start learning
    0.8
    EP 768 learning finished: 0.00722512001658604, 0.0020378165051569697
    769
    EP 769 simulation finished, start learning
    1.26
    EP 769 learning finished: 0.009548874796447393, 0.0019801186795741776
    770
    EP 770 simulation finished, start learning
    0.75
    EP 770 learning finished: 0.007100049038878176, 0.0019094777605640906
    771
    EP 771 simulation finished, start learning
    0.92
    EP 771 learning finished: 0.00637843970015517, 0.0019089010668986856
    772
    EP 772 simulation finished, start learning
    0.92
    EP 772 learning finished: 0.006323159309634008, 0.0017914609950994143
    773
    EP 773 simulation finished, start learning
    1.05
    EP 773 learning finished: 0.008830810704500536, 0.0019245597656803587
    774
    EP 774 simulation finished, start learning
    1.12
    EP 774 learning finished: 0.008773446117741557, 0.0018628384175976564
    775
    EP 775 simulation finished, start learning
    1.25
    EP 775 learning finished: 0.008721041896285087, 0.00186932658640096
    776
    EP 776 simulation finished, start learning
    0.92
    EP 776 learning finished: 0.007364140343195658, 0.0019398993394254965
    777
    EP 777 simulation finished, start learning
    1.05
    EP 777 learning finished: 0.011003144971485579, 0.0018384079840714912
    778
    EP 778 simulation finished, start learning
    1.08
    EP 778 learning finished: 0.01096251669353876, 0.001749184115531821
    779
    EP 779 simulation finished, start learning
    1.14
    EP 779 learning finished: 0.010923003032234924, 0.0016695663850203324
    780
    EP 780 simulation finished, start learning
    0.77
    EP 780 learning finished: 0.005447539752067243, 0.0015390859155488268
    781
    EP 781 simulation finished, start learning
    1.06
    EP 781 learning finished: 0.004490313348572528, 0.0015566422896941823
    782
    EP 782 simulation finished, start learning
    0.76
    EP 782 learning finished: 0.00471048052526335, 0.0017930348312872602
    783
    EP 783 simulation finished, start learning
    0.9
    EP 783 learning finished: 0.006007318702545561, 0.0020892243677119726
    784
    EP 784 simulation finished, start learning
    0.96
    EP 784 learning finished: 0.007181034075424364, 0.0020325657719449663
    785
    EP 785 simulation finished, start learning
    1.01
    EP 785 learning finished: 0.00591136642391632, 0.0019162606290242246
    786
    EP 786 simulation finished, start learning
    1.36
    EP 786 learning finished: 0.0054502222284913415, 0.001794068829574974
    787
    EP 787 simulation finished, start learning
    1.25
    EP 787 learning finished: 0.005447173617591616, 0.0016487116381198458
    788
    EP 788 simulation finished, start learning
    0.88
    EP 788 learning finished: 0.007093502387349737, 0.0018149758891554124
    789
    EP 789 simulation finished, start learning
    1.16
    EP 789 learning finished: 0.007036419486978751, 0.0018824716245345735
    790
    EP 790 simulation finished, start learning
    1.07
    EP 790 learning finished: 0.0069872524334702035, 0.0017345980407386385
    791
    EP 791 simulation finished, start learning
    0.87
    EP 791 learning finished: 0.00923304070386699, 0.0017658229538155823
    792
    EP 792 simulation finished, start learning
    0.51
    EP 792 learning finished: 0.005712774553318312, 0.0017716106783668972
    793
    EP 793 simulation finished, start learning
    0.79
    EP 793 learning finished: 0.005692675477718012, 0.0017985115560497367
    794
    EP 794 simulation finished, start learning
    0.81
    EP 794 learning finished: 0.006822494133850832, 0.0021305235516524135
    795
    EP 795 simulation finished, start learning
    1.03
    EP 795 learning finished: 0.00881767867713279, 0.0020529377444222507
    796
    EP 796 simulation finished, start learning
    1.38
    EP 796 learning finished: 0.010785769696722625, 0.0019751230624803495
    797
    EP 797 simulation finished, start learning
    0.98
    EP 797 learning finished: 0.016107507620102268, 0.0020987663064816753
    798
    EP 798 simulation finished, start learning
    1.09
    EP 798 learning finished: 0.01069044758727511, 0.0019736951167842742
    799
    EP 799 simulation finished, start learning
    1.16
    EP 799 learning finished: 0.010654189704083272, 0.001956242812919201
    800
    EP 800 simulation finished, start learning
    0.81
    EP 800 learning finished: 0.005395737722962313, 0.0018678186941664399
    801
    EP 801 simulation finished, start learning
    1.49
    EP 801 learning finished: 0.005619851578211765, 0.0016232475442335553
    802
    EP 802 simulation finished, start learning
    0.83
    EP 802 learning finished: 0.010196364312690775, 0.0015801945130562989
    803
    EP 803 simulation finished, start learning
    0.83
    EP 803 learning finished: 0.007348394854783891, 0.001739104065526631
    804
    EP 804 simulation finished, start learning
    0.6
    EP 804 learning finished: 0.0073063729580145065, 0.001969928612050059
    805
    EP 805 simulation finished, start learning
    0.73
    EP 805 learning finished: 0.007264241787204639, 0.0020809834374988234
    806
    EP 806 simulation finished, start learning
    0.96
    EP 806 learning finished: 0.0067380211505980725, 0.002166694159100563
    807
    EP 807 simulation finished, start learning
    0.84
    EP 807 learning finished: 0.008160890601871529, 0.0020837254385727005
    808
    EP 808 simulation finished, start learning
    0.98
    EP 808 learning finished: 0.008115204828804679, 0.0017714732736404115
    809
    EP 809 simulation finished, start learning
    0.88
    EP 809 learning finished: 0.008068218888514167, 0.0017547520522785132
    810
    EP 810 simulation finished, start learning
    0.83
    EP 810 learning finished: 0.0053834208570613384, 0.001760245910842631
    811
    EP 811 simulation finished, start learning
    0.92
    EP 811 learning finished: 0.010732750571527117, 0.0018625839101961694
    812
    EP 812 simulation finished, start learning
    1.23
    EP 812 learning finished: 0.01068712242156522, 0.0020231091625861203
    813
    EP 813 simulation finished, start learning
    0.92
    EP 813 learning finished: 0.015945958558708728, 0.0020244504674998692
    814
    EP 814 simulation finished, start learning
    1.0
    EP 814 learning finished: 0.015704299290477675, 0.002367357075365367
    815
    EP 815 simulation finished, start learning
    1.38
    EP 815 learning finished: 0.015621433937586193, 0.002114426553072646
    816
    EP 816 simulation finished, start learning
    1.21
    EP 816 learning finished: 0.01553705260908389, 0.0019838382341777205
    817
    EP 817 simulation finished, start learning
    1.14
    EP 817 learning finished: 0.020584872077840544, 0.0020355946668842717
    818
    EP 818 simulation finished, start learning
    0.9
    EP 818 learning finished: 0.005123457954043342, 0.0016687045247085058
    819
    EP 819 simulation finished, start learning
    0.83
    EP 819 learning finished: 0.007404852747580293, 0.0017414744696082202
    820
    EP 820 simulation finished, start learning
    0.53
    EP 820 learning finished: 0.010202208533988788, 0.002244966846377802
    821
    EP 821 simulation finished, start learning
    1.12
    EP 821 learning finished: 0.01016652093345538, 0.0022117633263732395
    822
    EP 822 simulation finished, start learning
    1.03
    EP 822 learning finished: 0.010132628206366068, 0.00192394772956124
    823
    EP 823 simulation finished, start learning
    0.92
    EP 823 learning finished: 0.01044519246171216, 0.0020046950318151208
    824
    EP 824 simulation finished, start learning
    1.09
    EP 824 learning finished: 0.010402316828283631, 0.0015116511411738254
    825
    EP 825 simulation finished, start learning
    0.81
    EP 825 learning finished: 0.010359993538563206, 0.0017661549061934734
    826
    EP 826 simulation finished, start learning
    1.05
    EP 826 learning finished: 0.0103156332692107, 0.002075673873137729
    827
    EP 827 simulation finished, start learning
    0.99
    EP 827 learning finished: 0.005144731366427457, 0.0018648846075941944
    828
    EP 828 simulation finished, start learning
    0.9
    EP 828 learning finished: 0.005947161915360155, 0.0019824188853806435
    829
    EP 829 simulation finished, start learning
    0.92
    EP 829 learning finished: 0.005341543804600946, 0.001800623628718051
    830
    EP 830 simulation finished, start learning
    1.01
    EP 830 learning finished: 0.005326252397221509, 0.002060233197350561
    831
    EP 831 simulation finished, start learning
    1.07
    EP 831 learning finished: 0.0051287063937728306, 0.002030918875790433
    832
    EP 832 simulation finished, start learning
    0.75
    EP 832 learning finished: 0.010228397009100831, 0.00209625928643678
    833
    EP 833 simulation finished, start learning
    0.83
    EP 833 learning finished: 0.010186002389614313, 0.002205984486260623
    834
    EP 834 simulation finished, start learning
    1.14
    EP 834 learning finished: 0.005077588179876358, 0.0016367735756754181
    835
    EP 835 simulation finished, start learning
    0.99
    EP 835 learning finished: 0.005415724141197842, 0.0018499431391510862
    836
    EP 836 simulation finished, start learning
    1.27
    EP 836 learning finished: 0.00494393169104379, 0.001605038110063975
    837
    EP 837 simulation finished, start learning
    1.18
    EP 837 learning finished: 0.004936476892543062, 0.0015069561529628683
    838
    EP 838 simulation finished, start learning
    1.18
    EP 838 learning finished: 0.005233764781747752, 0.001617125032573885
    839
    EP 839 simulation finished, start learning
    1.08
    EP 839 learning finished: 0.009831792974993814, 0.0015691957043267174
    840
    EP 840 simulation finished, start learning
    0.99
    EP 840 learning finished: 0.009798904439163092, 0.001817903852358596
    841
    EP 841 simulation finished, start learning
    1.19
    EP 841 learning finished: 0.010312688263629965, 0.0016918327822255524
    842
    EP 842 simulation finished, start learning
    0.73
    EP 842 learning finished: 0.00975212882611487, 0.002025590676175207
    843
    EP 843 simulation finished, start learning
    1.03
    EP 843 learning finished: 0.007639816902065277, 0.0019021315906612712
    844
    EP 844 simulation finished, start learning
    0.9
    EP 844 learning finished: 0.005045925202344037, 0.0016669260842010064
    845
    EP 845 simulation finished, start learning
    0.99
    EP 845 learning finished: 0.005038416493237552, 0.0015967687817110832
    846
    EP 846 simulation finished, start learning
    0.64
    EP 846 learning finished: 0.010239309527740659, 0.0014858442483231815
    847
    EP 847 simulation finished, start learning
    0.99
    EP 847 learning finished: 0.011342381000668311, 0.001779391857481591
    848
    EP 848 simulation finished, start learning
    1.59
    EP 848 learning finished: 0.011255638209439311, 0.0016660211049130819
    849
    EP 849 simulation finished, start learning
    1.14
    EP 849 learning finished: 0.011169153118510522, 0.001822342692859233
    850
    EP 850 simulation finished, start learning
    1.07
    EP 850 learning finished: 0.009617741670072899, 0.0015638574218578144
    851
    EP 851 simulation finished, start learning
    0.87
    EP 851 learning finished: 0.004796699653865755, 0.0012644378928935301
    852
    EP 852 simulation finished, start learning
    0.99
    EP 852 learning finished: 0.004790346615243912, 0.001394103995865037
    853
    EP 853 simulation finished, start learning
    1.23
    EP 853 learning finished: 0.009555030659334829, 0.0016912109447726892
    854
    EP 854 simulation finished, start learning
    0.79
    EP 854 learning finished: 0.012605410320551513, 0.0020892262891467756
    855
    EP 855 simulation finished, start learning
    0.98
    EP 855 learning finished: 0.01248543651788836, 0.0020859863114374017
    856
    EP 856 simulation finished, start learning
    1.09
    EP 856 learning finished: 0.014181315103863223, 0.0022729550288924927
    857
    EP 857 simulation finished, start learning
    0.92
    EP 857 learning finished: 0.009415056446665293, 0.0019885871710132313
    858
    EP 858 simulation finished, start learning
    0.94
    EP 858 learning finished: 0.009386856559338685, 0.0017004293081412658
    859
    EP 859 simulation finished, start learning
    1.27
    EP 859 learning finished: 0.009356928272423737, 0.0018042997357581858
    860
    EP 860 simulation finished, start learning
    1.01
    EP 860 learning finished: 0.005002956705457251, 0.0017627288422200904
    861
    EP 861 simulation finished, start learning
    0.88
    EP 861 learning finished: 0.009312474677715787, 0.0019938117430470387
    862
    EP 862 simulation finished, start learning
    1.08
    EP 862 learning finished: 0.00928200083964148, 0.0019738320082425542
    863
    EP 863 simulation finished, start learning
    0.79
    EP 863 learning finished: 0.009252606151523679, 0.0020091661966625263
    864
    EP 864 simulation finished, start learning
    1.18
    EP 864 learning finished: 0.009222952310212307, 0.0018908345484156541
    865
    EP 865 simulation finished, start learning
    0.72
    EP 865 learning finished: 0.004792093143320741, 0.0015583093799542425
    866
    EP 866 simulation finished, start learning
    0.86
    EP 866 learning finished: 0.0046327382817210605, 0.0015946431465080346
    867
    EP 867 simulation finished, start learning
    0.88
    EP 867 learning finished: 0.009183866728026846, 0.0015795461327636345
    868
    EP 868 simulation finished, start learning
    0.79
    EP 868 learning finished: 0.009153958566141984, 0.0016396643649307374
    869
    EP 869 simulation finished, start learning
    1.03
    EP 869 learning finished: 0.00990903867116888, 0.0017769981436952381
    870
    EP 870 simulation finished, start learning
    0.75
    EP 870 learning finished: 0.009871057065667266, 0.0017420541831231623
    871
    EP 871 simulation finished, start learning
    0.73
    EP 871 learning finished: 0.019623873404310377, 0.001980496677097085
    872
    EP 872 simulation finished, start learning
    1.11
    EP 872 learning finished: 0.014610645185987092, 0.0021060811256458783
    873
    EP 873 simulation finished, start learning
    0.94
    EP 873 learning finished: 0.01452377102430491, 0.0019771584369646378
    874
    EP 874 simulation finished, start learning
    1.55
    EP 874 learning finished: 0.019226184370916943, 0.001963364499544498
    875
    EP 875 simulation finished, start learning
    0.93
    EP 875 learning finished: 0.009555056072356294, 0.0016520360105785337
    876
    EP 876 simulation finished, start learning
    0.97
    EP 876 learning finished: 0.009516936137775822, 0.0015965185172931717
    877
    EP 877 simulation finished, start learning
    0.82
    EP 877 learning finished: 0.009479544077780844, 0.0018034873636838533
    878
    EP 878 simulation finished, start learning
    1.07
    EP 878 learning finished: 0.004763677706353775, 0.0017100013133639347
    879
    EP 879 simulation finished, start learning
    1.01
    EP 879 learning finished: 0.004747672023659888, 0.0017310516684716226
    880
    EP 880 simulation finished, start learning
    1.27
    EP 880 learning finished: 0.0047070741384551695, 0.0014817255073714952
    881
    EP 881 simulation finished, start learning
    0.9
    EP 881 learning finished: 0.004526005116855192, 0.0013868130492488279
    882
    EP 882 simulation finished, start learning
    0.55
    EP 882 learning finished: 0.00903050806344785, 0.0016666845286230775
    883
    EP 883 simulation finished, start learning
    1.23
    EP 883 learning finished: 0.009001892632283237, 0.0015925249100467165
    884
    EP 884 simulation finished, start learning
    1.03
    EP 884 learning finished: 0.013450013879007905, 0.0017884123923299355
    885
    EP 885 simulation finished, start learning
    0.96
    EP 885 learning finished: 0.008932052270488988, 0.0017710161550699155
    886
    EP 886 simulation finished, start learning
    0.96
    EP 886 learning finished: 0.009376844555446517, 0.0015890505211593325
    887
    EP 887 simulation finished, start learning
    1.27
    EP 887 learning finished: 0.009340333882717822, 0.0015706532762525816
    888
    EP 888 simulation finished, start learning
    1.14
    EP 888 learning finished: 0.009305604879565443, 0.0013721063795655297
    889
    EP 889 simulation finished, start learning
    0.84
    EP 889 learning finished: 0.004639724944603962, 0.001201424054107919
    890
    EP 890 simulation finished, start learning
    1.19
    EP 890 learning finished: 0.0022469276641352565, 0.0010615557532938913
    891
    EP 891 simulation finished, start learning
    0.98
    EP 891 learning finished: 0.004633264325644149, 0.001254450792262718
    892
    EP 892 simulation finished, start learning
    1.12
    EP 892 learning finished: 0.0056699456945565885, 0.0014913409033213976
    893
    EP 893 simulation finished, start learning
    1.09
    EP 893 learning finished: 0.005652306914733174, 0.0015707550411267757
    894
    EP 894 simulation finished, start learning
    0.96
    EP 894 learning finished: 0.004828274131071453, 0.001661768795683291
    895
    EP 895 simulation finished, start learning
    1.21
    EP 895 learning finished: 0.00988874785878835, 0.0015741158834427782
    896
    EP 896 simulation finished, start learning
    1.05
    EP 896 learning finished: 0.007856200207697839, 0.0014615941960942902
    897
    EP 897 simulation finished, start learning
    1.2
    EP 897 learning finished: 0.007808030382651765, 0.0014836815318582108
    898
    EP 898 simulation finished, start learning
    1.01
    EP 898 learning finished: 0.0077624666540465626, 0.0014180587530144978
    899
    EP 899 simulation finished, start learning
    1.01
    EP 899 learning finished: 0.004407851016647424, 0.0013617213242428788
    900
    EP 900 simulation finished, start learning
    0.98
    EP 900 learning finished: 0.004709525614916998, 0.001543609521732242
    901
    EP 901 simulation finished, start learning
    0.84
    EP 901 learning finished: 0.0046935176247480115, 0.0014998095625638197
    902
    EP 902 simulation finished, start learning
    1.34
    EP 902 learning finished: 0.00467718831293773, 0.0014310629310492207
    903
    EP 903 simulation finished, start learning
    0.99
    EP 903 learning finished: 0.009145010435646306, 0.0015075935905225213
    904
    EP 904 simulation finished, start learning
    0.9
    EP 904 learning finished: 0.009111699988431532, 0.0015752558730822985
    905
    EP 905 simulation finished, start learning
    1.0
    EP 905 learning finished: 0.009075096456412801, 0.0015816843655749452
    906
    EP 906 simulation finished, start learning
    0.55
    EP 906 learning finished: 0.009041156513406534, 0.0019052697239207994
    907
    EP 907 simulation finished, start learning
    1.0
    EP 907 learning finished: 0.006515445022662547, 0.0018530076565216463
    908
    EP 908 simulation finished, start learning
    1.14
    EP 908 learning finished: 0.007540386491275086, 0.001833474079557057
    909
    EP 909 simulation finished, start learning
    1.11
    EP 909 learning finished: 0.007685376037160907, 0.002008242145441208
    910
    EP 910 simulation finished, start learning
    0.62
    EP 910 learning finished: 0.00763975641141279, 0.0020132047972750283
    911
    EP 911 simulation finished, start learning
    0.9
    EP 911 learning finished: 0.008734451506691165, 0.0020885971980745266
    912
    EP 912 simulation finished, start learning
    1.14
    EP 912 learning finished: 0.008708989393228617, 0.0017855163541657254
    913
    EP 913 simulation finished, start learning
    0.94
    EP 913 learning finished: 0.008682909441383835, 0.0015726487557097268
    914
    EP 914 simulation finished, start learning
    1.16
    EP 914 learning finished: 0.005007885304786797, 0.0014331675501635388
    915
    EP 915 simulation finished, start learning
    0.83
    EP 915 learning finished: 0.006468001921859923, 0.0013163670262907323
    916
    EP 916 simulation finished, start learning
    0.96
    EP 916 learning finished: 0.006429801349664643, 0.0015777804427117847
    917
    EP 917 simulation finished, start learning
    1.18
    EP 917 learning finished: 0.008892960362981484, 0.0018681995510790306
    918
    EP 918 simulation finished, start learning
    1.07
    EP 918 learning finished: 0.006803804488940912, 0.0017688469493938597
    919
    EP 919 simulation finished, start learning
    0.77
    EP 919 learning finished: 0.008844159614959324, 0.0018839691117232549
    920
    EP 920 simulation finished, start learning
    1.25
    EP 920 learning finished: 0.008810820020211274, 0.0015724486068131038
    921
    EP 921 simulation finished, start learning
    1.05
    EP 921 learning finished: 0.008775668022103389, 0.001446980977344442
    922
    EP 922 simulation finished, start learning
    0.81
    EP 922 learning finished: 0.008743515718957262, 0.0016817685477699823
    923
    EP 923 simulation finished, start learning
    0.92
    EP 923 learning finished: 0.00436005186142463, 0.0016167802433926515
    924
    EP 924 simulation finished, start learning
    0.9
    EP 924 learning finished: 0.00893576500116744, 0.001938459516864971
    925
    EP 925 simulation finished, start learning
    0.88
    EP 925 learning finished: 0.008874288703911049, 0.0017796544431087635
    926
    EP 926 simulation finished, start learning
    1.13
    EP 926 learning finished: 0.008811496734779212, 0.0018392970117593055
    927
    EP 927 simulation finished, start learning
    0.87
    EP 927 learning finished: 0.00875327365993428, 0.0019072775780014373
    928
    EP 928 simulation finished, start learning
    1.03
    EP 928 learning finished: 0.004799993602127017, 0.001700713096880968
    929
    EP 929 simulation finished, start learning
    0.94
    EP 929 learning finished: 0.00865533030580945, 0.0016873384156730342
    930
    EP 930 simulation finished, start learning
    0.9
    EP 930 learning finished: 0.012925423728459506, 0.0015681011519714239
    931
    EP 931 simulation finished, start learning
    0.88
    EP 931 learning finished: 0.008577772644255077, 0.0014036781705571988
    932
    EP 932 simulation finished, start learning
    1.12
    EP 932 learning finished: 0.017059073822998982, 0.0017551573326483605
    933
    EP 933 simulation finished, start learning
    0.92
    EP 933 learning finished: 0.012709438152903374, 0.0019186970739965975
    934
    EP 934 simulation finished, start learning
    1.34
    EP 934 learning finished: 0.00859746868497302, 0.0017967222346654405
    935
    EP 935 simulation finished, start learning
    0.77
    EP 935 learning finished: 0.008572271311573232, 0.0016938054158960048
    936
    EP 936 simulation finished, start learning
    1.11
    EP 936 learning finished: 0.012808504288836176, 0.0013949600663809292
    937
    EP 937 simulation finished, start learning
    0.73
    EP 937 learning finished: 0.004362927269013994, 0.0015877114651430546
    938
    EP 938 simulation finished, start learning
    0.9
    EP 938 learning finished: 0.00642066045130818, 0.0017278906285608797
    939
    EP 939 simulation finished, start learning
    0.55
    EP 939 learning finished: 0.008321372420731166, 0.00199564328973785
    940
    EP 940 simulation finished, start learning
    0.7
    EP 940 learning finished: 0.008247437503369426, 0.001988065473501305
    941
    EP 941 simulation finished, start learning
    1.03
    EP 941 learning finished: 0.008171579255904504, 0.0016043882476626248
    942
    EP 942 simulation finished, start learning
    1.04
    EP 942 learning finished: 0.008096726365540263, 0.001444756549751236
    943
    EP 943 simulation finished, start learning
    1.03
    EP 943 learning finished: 0.004167684069924071, 0.0013285634225362053
    944
    EP 944 simulation finished, start learning
    0.96
    EP 944 learning finished: 0.008313189542212385, 0.0014823580972238848
    945
    EP 945 simulation finished, start learning
    1.09
    EP 945 learning finished: 0.00828351450147391, 0.0016141658545181653
    946
    EP 946 simulation finished, start learning
    0.97
    EP 946 learning finished: 0.016474725724537898, 0.0018166706983921282
    947
    EP 947 simulation finished, start learning
    1.0
    EP 947 learning finished: 0.012276252394702425, 0.0017363169701066772
    948
    EP 948 simulation finished, start learning
    0.8
    EP 948 learning finished: 0.008147588533581107, 0.001602789163065832
    949
    EP 949 simulation finished, start learning
    0.92
    EP 949 learning finished: 0.012165590733262732, 0.0016799250464603066
    950
    EP 950 simulation finished, start learning
    1.39
    EP 950 learning finished: 0.008074372334397673, 0.0016493640417691025
    951
    EP 951 simulation finished, start learning
    0.96
    EP 951 learning finished: 0.008044824613796742, 0.0016755049138001965
    952
    EP 952 simulation finished, start learning
    0.75
    EP 952 learning finished: 0.00801911919095577, 0.0017288090179167354
    953
    EP 953 simulation finished, start learning
    0.94
    EP 953 learning finished: 0.007254066396894743, 0.001563789756737946
    954
    EP 954 simulation finished, start learning
    1.18
    EP 954 learning finished: 0.0084142000063121, 0.0014821903432806205
    955
    EP 955 simulation finished, start learning
    0.88
    EP 955 learning finished: 0.008387802715830173, 0.0014236064005953702
    956
    EP 956 simulation finished, start learning
    0.9
    EP 956 learning finished: 0.004184513821590574, 0.0013574061649136576
    957
    EP 957 simulation finished, start learning
    0.94
    EP 957 learning finished: 0.007972131502881119, 0.0014984024004517926
    958
    EP 958 simulation finished, start learning
    1.44
    EP 958 learning finished: 0.007943482860567563, 0.0013683828929185737
    959
    EP 959 simulation finished, start learning
    0.99
    EP 959 learning finished: 0.007915397469803409, 0.0016409544191329783
    960
    EP 960 simulation finished, start learning
    1.07
    EP 960 learning finished: 0.007752622912793328, 0.0016924694633350945
    961
    EP 961 simulation finished, start learning
    1.01
    EP 961 learning finished: 0.007132594080159649, 0.001701050179542224
    962
    EP 962 simulation finished, start learning
    1.11
    EP 962 learning finished: 0.007100150014540674, 0.0018317731118712988
    963
    EP 963 simulation finished, start learning
    1.2
    EP 963 learning finished: 0.007068681010759192, 0.0017898432362824374
    964
    EP 964 simulation finished, start learning
    1.07
    EP 964 learning finished: 0.007037437257125134, 0.0016022000404928016
    965
    EP 965 simulation finished, start learning
    1.03
    EP 965 learning finished: 0.005979011162420278, 0.0014433588139454723
    966
    EP 966 simulation finished, start learning
    1.05
    EP 966 learning finished: 0.004946605810701499, 0.001438613290224544
    967
    EP 967 simulation finished, start learning
    1.27
    EP 967 learning finished: 0.004306819529555028, 0.0012534980238390891
    968
    EP 968 simulation finished, start learning
    0.88
    EP 968 learning finished: 0.006433616504963879, 0.0014723408846238628
    969
    EP 969 simulation finished, start learning
    0.6
    EP 969 learning finished: 0.008528629125375886, 0.0018250838248416723
    970
    EP 970 simulation finished, start learning
    0.94
    EP 970 learning finished: 0.008858036149103299, 0.0017965580947758297
    971
    EP 971 simulation finished, start learning
    0.81
    EP 971 learning finished: 0.008793819647628709, 0.001879325074942706
    972
    EP 972 simulation finished, start learning
    1.07
    EP 972 learning finished: 0.006989228804529901, 0.0016466034470542235
    973
    EP 973 simulation finished, start learning
    0.92
    EP 973 learning finished: 0.004137969838619249, 0.0013016829526714124
    974
    EP 974 simulation finished, start learning
    0.73
    EP 974 learning finished: 0.006915103754971952, 0.0013895035603421202
    975
    EP 975 simulation finished, start learning
    0.92
    EP 975 learning finished: 0.006886996344089624, 0.0012493916525423976
    976
    EP 976 simulation finished, start learning
    0.75
    EP 976 learning finished: 0.006857645209780827, 0.0011913650061519954
    977
    EP 977 simulation finished, start learning
    0.93
    EP 977 learning finished: 0.006937145744338125, 0.001303898472899745
    978
    EP 978 simulation finished, start learning
    0.92
    EP 978 learning finished: 0.0034636862328927265, 0.0012112900284285966
    979
    EP 979 simulation finished, start learning
    1.18
    EP 979 learning finished: 0.004137377497728245, 0.0014749465497329028
    980
    EP 980 simulation finished, start learning
    0.92
    EP 980 learning finished: 0.008256399813044712, 0.001705961011053178
    981
    EP 981 simulation finished, start learning
    0.92
    EP 981 learning finished: 0.008231554921060091, 0.0016802049644177447
    982
    EP 982 simulation finished, start learning
    0.95
    EP 982 learning finished: 0.008206416580541998, 0.001659062061917016
    983
    EP 983 simulation finished, start learning
    1.27
    EP 983 learning finished: 0.00409362703544458, 0.001363013243208408
    984
    EP 984 simulation finished, start learning
    0.99
    EP 984 learning finished: 0.004087978991460073, 0.0015594719914894112
    985
    EP 985 simulation finished, start learning
    1.21
    EP 985 learning finished: 0.005704659271041024, 0.0016586690955982253
    986
    EP 986 simulation finished, start learning
    1.43
    EP 986 learning finished: 0.005734171241503105, 0.0016671971032459626
    987
    EP 987 simulation finished, start learning
    1.23
    EP 987 learning finished: 0.008133067364961555, 0.0018959188695835683
    988
    EP 988 simulation finished, start learning
    1.06
    EP 988 learning finished: 0.0056852378475138465, 0.0017826530114595923
    989
    EP 989 simulation finished, start learning
    0.92
    EP 989 learning finished: 0.006335034041939025, 0.0016367199202508842
    990
    EP 990 simulation finished, start learning
    1.12
    EP 990 learning finished: 0.006295035179274322, 0.001625392858983457
    991
    EP 991 simulation finished, start learning
    0.81
    EP 991 learning finished: 0.005214928837157018, 0.0015788256033013731
    992
    EP 992 simulation finished, start learning
    0.64
    EP 992 learning finished: 0.004438852362657286, 0.0015417061648890923
    993
    EP 993 simulation finished, start learning
    0.97
    EP 993 learning finished: 0.004416408672936356, 0.0016575409887409909
    994
    EP 994 simulation finished, start learning
    0.9
    EP 994 learning finished: 0.0061061601612824035, 0.0018085135078198672
    995
    EP 995 simulation finished, start learning
    0.62
    EP 995 learning finished: 0.007705439715956608, 0.002122301295032894
    996
    EP 996 simulation finished, start learning
    0.92
    EP 996 learning finished: 0.0076814506454656595, 0.0019787252674613176
    997
    EP 997 simulation finished, start learning
    0.79
    EP 997 learning finished: 0.007655715759324444, 0.001899173793411299
    998
    EP 998 simulation finished, start learning
    0.86
    EP 998 learning finished: 0.00799576315924666, 0.0018892519405580982
    999
    EP 999 simulation finished, start learning
    0.94
    EP 999 learning finished: 0.007972000220366038, 0.0015988397114316127



```python
v_opt, pi_opt= q_opt.q.yield_v_pi()
```


```python
v_opt
```




    0    12.284256
    1    15.304478
    2    18.505131
    3    21.847240
    4    25.338371
    5    18.516494
    6    15.350039
    7    12.305080
    dtype: float64




```python
display_pi(pi_opt)
```

    
        -----   -----   -----
        | d |   | u |   | d |
        ---------------------
        | r | r | u | l | l |
        ---------------------
        



```python
for s in state_space:
    for a in control_space:
        print(f'state: {s}, action_buffer: {[a]}, OAS: {pi_opt((s,a))}, planning: {pi_model((s, a))}, AMDP: {pi_AMDP_((s,a))}')
        
pi_opt = lambda x: get_opt_act(q_opt.q, model, x[0], [x[1]])
```

    state: 0, action_buffer: [0], OAS: 1, planning: 1, AMDP: 1
    state: 0, action_buffer: [1], OAS: 3, planning: 3, AMDP: 3
    state: 0, action_buffer: [2], OAS: 1, planning: 1, AMDP: 1
    state: 0, action_buffer: [3], OAS: 1, planning: 1, AMDP: 1
    state: 1, action_buffer: [0], OAS: 1, planning: 1, AMDP: 1
    state: 1, action_buffer: [1], OAS: 3, planning: 3, AMDP: 3
    state: 1, action_buffer: [2], OAS: 3, planning: 3, AMDP: 3
    state: 1, action_buffer: [3], OAS: 3, planning: 3, AMDP: 3
    state: 2, action_buffer: [0], OAS: 3, planning: 3, AMDP: 3
    state: 2, action_buffer: [1], OAS: 3, planning: 3, AMDP: 3
    state: 2, action_buffer: [2], OAS: 3, planning: 3, AMDP: 3
    state: 2, action_buffer: [3], OAS: 0, planning: 0, AMDP: 0
    state: 3, action_buffer: [0], OAS: 0, planning: 0, AMDP: 0
    state: 3, action_buffer: [1], OAS: 0, planning: 0, AMDP: 0
    state: 3, action_buffer: [2], OAS: 3, planning: 3, AMDP: 3
    state: 3, action_buffer: [3], OAS: 2, planning: 2, AMDP: 2
    state: 4, action_buffer: [0], OAS: 3, planning: 1, AMDP: 0
    state: 4, action_buffer: [1], OAS: 0, planning: 0, AMDP: 0
    state: 4, action_buffer: [2], OAS: 0, planning: 0, AMDP: 0
    state: 4, action_buffer: [3], OAS: 0, planning: 0, AMDP: 0
    state: 5, action_buffer: [0], OAS: 2, planning: 2, AMDP: 2
    state: 5, action_buffer: [1], OAS: 2, planning: 2, AMDP: 2
    state: 5, action_buffer: [2], OAS: 0, planning: 0, AMDP: 0
    state: 5, action_buffer: [3], OAS: 2, planning: 2, AMDP: 2
    state: 6, action_buffer: [0], OAS: 1, planning: 1, AMDP: 1
    state: 6, action_buffer: [1], OAS: 2, planning: 2, AMDP: 2
    state: 6, action_buffer: [2], OAS: 2, planning: 2, AMDP: 2
    state: 6, action_buffer: [3], OAS: 2, planning: 2, AMDP: 2
    state: 7, action_buffer: [0], OAS: 1, planning: 1, AMDP: 1
    state: 7, action_buffer: [1], OAS: 2, planning: 2, AMDP: 2
    state: 7, action_buffer: [2], OAS: 1, planning: 1, AMDP: 1
    state: 7, action_buffer: [3], OAS: 1, planning: 1, AMDP: 1



```python
q_opt.q
```




                0          1          2
    -1  25.622767  36.756796  50.515048
     1  41.316134  45.395789  35.990414




```python
def eval_pi_model(env, q, model, discount_factor, MAX_ITER = 1000000):

    path_len = int(np.log(1e-3)/np.log(discount_factor))
    path_num = int(MAX_ITER/path_len)
    obs = 0
    disc_arr = np.array([discount_factor**n for n in range(path_len)])
    r_ls = np.array([[np.nan]*path_len for _ in range(path_num)])
    ret = []
    delay_step = env.num
    action_buffer = [-1]*delay_step
    for k in range(path_num):
        obs = env.reset(random= True)
        for i in range(path_len):
            action = get_opt_act(q, model, obs, action_buffer)
            action_buffer.pop(0)
            action_buffer.append(action)
            next_obs, reward, _, _ = env.step(action)
    #         discounted_cum_r = reward + discount_factor*discounted_cum_r
            r_ls[k, i] = reward
            obs = next_obs
        ret.append(np.cumsum(disc_arr*r_ls[k, :]))
#     return np.mean(ret) ,ret
    return np.mean(r_ls), r_ls.mean(axis = 1)
```


```python
val, v_samples_lin = eval_pi_model(env, q_opt.q, model, 0.95)
val
```




    1.5354082575596955




```python
exp2_opt = v_samples_lin
```


```python
from matplotlib.pyplot import figure

figure(figsize=(8, 6), dpi=100)
sns.distplot(exp2_opt, label = 'Opt. Action Selection')
sns.distplot(exp2_model, label = 'model')
sns.distplot(exp2_random, label = 'random')
sns.distplot(exp1_opt, label = 'Augmented MDP')
plt.xlabel("average reward")
plt.ylabel("freq.")
plt.legend()
# plt.set_size_inches(18.5, 10.5)
plt.savefig('exp2.png', dpi=100)
```


```python
plt.savefig('exp1.png')
```


    <Figure size 432x288 with 0 Axes>



```python
# For a given policy function, we solve the system A^{\pi}x=b^{\pi} to get the optimal value function

def linear_system(pi, trans_prob, reward, state_space, gamma):
    n = len(state_space)
    A = np.zeros(shape = (n,n))
    b = np.zeros(n)
    for i, s in enumerate(state_space):
        for j, next_s in enumerate(state_space):
            A[i,j] = trans_prob(next_s, s, pi(s))
            
        b[i] = -reward(s, pi(s))
        print(f'policy({s}): {pi(s)}')
    A = gamma*A-np.eye(n)
    return A, b
        
```


```python
from numpy.linalg import solve
```


```python
# policy of opt action selection
A, b = linear_system(pi_opt, trans_prob_d, reward_d,  state_space_d, alpha)
A, b
```

    policy((0, -1)): 1
    policy((0, 1)): 1
    policy((1, -1)): 1
    policy((1, 1)): -1
    policy((2, -1)): 1
    policy((2, 1)): -1





    (array([[-1.  ,  0.95,  0.  ,  0.  ,  0.  ,  0.  ],
            [ 0.  , -1.  ,  0.  ,  0.95,  0.  ,  0.  ],
            [ 0.  ,  0.76, -1.  ,  0.  ,  0.  ,  0.19],
            [ 0.57,  0.  ,  0.  , -1.  ,  0.38,  0.  ],
            [ 0.  ,  0.  ,  0.  ,  0.95, -1.  ,  0.  ],
            [ 0.  ,  0.  ,  0.  ,  0.  ,  0.95, -1.  ]]),
     array([  1.,  -1.,  -0.,  -0., -10.,  10.]))




```python
solve(A, b)
```




    array([27.27312864, 29.76118804, 27.71489735, 30.27493478, 38.76118804,
           26.82312864])




```python
pi_AMDP_func = lambda x:pi_AMDP[x]
```


```python
# policy of AMDP

A, b = linear_system(pi_AMDP_func, trans_prob_d, reward_d,  state_space_d, alpha)
A, b
```

    policy((0, -1)): 1
    policy((0, 1)): 1
    policy((1, -1)): -1
    policy((1, 1)): -1
    policy((2, -1)): 1
    policy((2, 1)): -1





    (array([[-1.  ,  0.95,  0.  ,  0.  ,  0.  ,  0.  ],
            [ 0.  , -1.  ,  0.  ,  0.95,  0.  ,  0.  ],
            [ 0.76,  0.  , -1.  ,  0.  ,  0.19,  0.  ],
            [ 0.57,  0.  ,  0.  , -1.  ,  0.38,  0.  ],
            [ 0.  ,  0.  ,  0.  ,  0.95, -1.  ,  0.  ],
            [ 0.  ,  0.  ,  0.  ,  0.  ,  0.95, -1.  ]]),
     array([  1.,  -1.,  -0.,  -0., -10.,  10.]))




```python
solve(A, b)
```




    array([27.27312864, 29.76118804, 28.09220349, 30.27493478, 38.76118804,
           26.82312864])




```python
# policy of planning based mathod
```


```python
A, b = linear_system(pi_model, trans_prob_d, reward_d,  state_space_d, alpha)
A, b
```

    policy((0, -1)): 1
    policy((0, 1)): 1
    policy((1, -1)): 1
    policy((1, 1)): 1
    policy((2, -1)): 1
    policy((2, 1)): -1





    (array([[-1.  ,  0.95,  0.  ,  0.  ,  0.  ,  0.  ],
            [ 0.  , -1.  ,  0.  ,  0.95,  0.  ,  0.  ],
            [ 0.  ,  0.76, -1.  ,  0.  ,  0.  ,  0.19],
            [ 0.  ,  0.57,  0.  , -1.  ,  0.  ,  0.38],
            [ 0.  ,  0.  ,  0.  ,  0.95, -1.  ,  0.  ],
            [ 0.  ,  0.  ,  0.  ,  0.  ,  0.95, -1.  ]]),
     array([  1.,  -1.,  -0.,  -0., -10.,  10.]))




```python
solve(A, b)
```




    array([ 2.91797923,  4.12418866,  3.60329944,  3.28861965, 13.12418866,
            2.46797923])




```python

```


```python

```


```python

```


```python

```


```python
pi_d
```




    (0, -1)    1
    (0, 1)     1
    (1, -1)   -1
    (1, 1)    -1
    (2, -1)    1
    (2, 1)    -1
    dtype: int64




```python
def eval_pi_model(env, pi, discount_factor, MAX_ITER = 1000000):

    path_len = int(np.log(1e-3)/np.log(discount_factor))
    path_num = int(MAX_ITER/path_len)
    obs = 0
    disc_arr = np.array([discount_factor**n for n in range(path_len)])
    r_ls = np.array([[np.nan]*path_len for _ in range(path_num)])
    ret = []
    delay_step = env.num
    action_buffer = [-1]*delay_step
    for k in range(path_num):
        obs = env.reset(random= True)
        for i in range(path_len):
            action = pi[(obs, action_buffer[0])]
            action_buffer.pop(0)
            action_buffer.append(action)
            next_obs, reward, _, _ = env.step(action)
    #         discounted_cum_r = reward + discount_factor*discounted_cum_r
            r_ls[k, i] = reward
            obs = next_obs
        ret.append(np.cumsum(disc_arr*r_ls[k, :]))
#     return np.mean(ret) ,ret
    return np.mean(r_ls), r_ls.mean(axis = 1)
```


```python
val, v_samples_lin = eval_pi_model(env, pi_d, 0.95)
val
```




    1.5329450309428467




```python

```


```python

```


```python

```


```python

```


```python
# Q-learning using LEARNING-BASED model, async reward collection and optimal action selection



class Q_learning:
    def __init__(self, q, env, delay_step, model, model_wrapper, epsilon = 0.1, MAX_ITER = 10000, episode_length = 100, threshold = 1e-3):
        self.q = q
        self.env = env
        self.delay_step = delay_step
        self.epsilon = epsilon
        self.MAX_ITER = MAX_ITER
        self.episode_length = episode_length
        self.threshold = threshold
        self.pool = Pool(400)
        self.qs = []
        self.action_memory = [-1]*delay_step
        self.model = model
        self.model_wrapper = model_wrapper
        
    def train(self, lr = lambda *args: 0.1):
        q = self.q
        env = self.env
        epsilon = self.epsilon
        MAX_ITER = self.MAX_ITER
        episode_length = self.episode_length
        threshold = self.threshold
        
        for i in range(MAX_ITER):
            print(i)
            obs = env.reset(random = True)
            r_ls = []
            model = self.model_wrapper(self.model)
            for _ in range(100):
                
                
                action, is_random = (np.random.choice(control_space), True) \
                                    if np.random.uniform() < epsilon else \
                                    (get_opt_act(q, model, obs, self.action_memory), False)
                last_action = self.action_memory.pop(0)
                self.action_memory.append(action)
                next_obs, reward, _, _ = env.step(action)
                self.pool.append((obs, next_obs, last_action, reward, is_random))
                r_ls.append(reward)
                obs = next_obs
            if not self.pool.isFull:
                continue
            print(f'EP {i} simulation finished, start learning')
            q_new = self.update_q(q, self.pool ,i, 0.95, lr)
            self.update_model(self.pool)
            self.q = q_new
            print(np.mean(r_ls))
            self.qs.append(q_new)
            print(f'EP {i} learning finished: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
            if q_new.difference(q).L_1() < threshold and False:
                break
            else:
                q = q_new
        return q_new
    
    def update_model(self, pool):
        tmp = []
        for ele in pool:
            tmp.append(ele)
    
        x_train = np.array(tmp)[:, [0,2]]
        y_train = np.array(tmp)[:, 1]
        self.model.fit(x_train, y_train, batch_size=64, epochs=3, validation_split=0.2)
        
    
    def update_q(self, q, pool, t ,alpha, lr):
        pool = pool.shuffle()
        for _ in range(1):
            q = deepcopy(q)
#           
            for s, next_s, a, r, is_random in pool:
#                 if is_random: continue
                q[s, a] = q[s,a] + lr(t, s, a)*(r + alpha*q.max_value_act(next_s)[0]-q[s,a])
#             print(f'Learning ep: {i}: {q_new.difference(q).L_infty()}, {q_new.difference(q).L_1()}')
#             if q_new.difference(q).L_1() < self.threshold:
#                 break
#             else:
#                 q = q_new
        return q

def get_opt_act(q,model, x, acts):
    act_idx = np.argmax([np.sum([q[s, u]*model(s, x, acts) for s in q.state_space]) for u in q.control_space])
    return q.control_space[act_idx]
```


```python
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
```


```python
inputs =  keras.Input(shape=(2,))

layer1 = layers.Dense(64, activation="relu", name="layer1")(inputs)
layer1 = layers.Dropout(0.5)(layer1)
layer2 = layers.Dense(64, activation="relu", name="layer2")(layer1)
layer2 = layers.Dropout(0.5)(layer2)
outputs = layers.Dense(3, activation="softmax", name="output_layer")(layer2)

model = keras.Model(inputs = inputs, outputs = outputs)
model.compile(
    loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    optimizer=keras.optimizers.RMSprop(),
    metrics=["accuracy"],
)


model.summary()
```

    Model: "model"
    _________________________________________________________________
    Layer (type)                 Output Shape              Param #   
    =================================================================
    input_1 (InputLayer)         [(None, 2)]               0         
    _________________________________________________________________
    layer1 (Dense)               (None, 64)                192       
    _________________________________________________________________
    dropout (Dropout)            (None, 64)                0         
    _________________________________________________________________
    layer2 (Dense)               (None, 64)                4160      
    _________________________________________________________________
    dropout_1 (Dropout)          (None, 64)                0         
    _________________________________________________________________
    output_layer (Dense)         (None, 3)                 195       
    =================================================================
    Total params: 4,547
    Trainable params: 4,547
    Non-trainable params: 0
    _________________________________________________________________



```python
tmp = []
for ele in q_learn.pool:
    tmp.append(ele)
    
x_train = np.array(tmp)[:, [0,2]]
y_train = np.array(tmp)[:, 1]
def one_hot_encode(y):
    y = y.astype(int)
    elements = set(y)
    n = len(elements)
    ret = np.zeros((len(y), n))
    ret[np.arange(y.size),y] = 1
    return ret
# y_train = one_hot_encode(y_train)
print(x_train.shape, y_train.shape)
```

    (400, 2) (400,)



```python
model.predict([[1, 1]])[0,0]
```




    0.28935617




```python
history = model.fit(x_train, y_train, batch_size=64, epochs=10, validation_split=0.2)
```

    Train on 320 samples, validate on 80 samples
    Epoch 1/10
    320/320 [==============================] - 0s 288us/sample - loss: 0.7200 - accuracy: 0.8156 - val_loss: 0.7028 - val_accuracy: 0.8000
    Epoch 2/10
    320/320 [==============================] - 0s 196us/sample - loss: 0.7178 - accuracy: 0.8250 - val_loss: 0.7029 - val_accuracy: 0.8000
    Epoch 3/10
    320/320 [==============================] - 0s 193us/sample - loss: 0.7206 - accuracy: 0.8156 - val_loss: 0.7034 - val_accuracy: 0.8000
    Epoch 4/10
    320/320 [==============================] - 0s 189us/sample - loss: 0.7223 - accuracy: 0.7937 - val_loss: 0.7030 - val_accuracy: 0.8000
    Epoch 5/10
    320/320 [==============================] - 0s 194us/sample - loss: 0.7185 - accuracy: 0.8156 - val_loss: 0.7026 - val_accuracy: 0.8000
    Epoch 6/10
    320/320 [==============================] - 0s 191us/sample - loss: 0.7228 - accuracy: 0.8094 - val_loss: 0.7036 - val_accuracy: 0.8000
    Epoch 7/10
    320/320 [==============================] - 0s 195us/sample - loss: 0.7166 - accuracy: 0.8094 - val_loss: 0.7030 - val_accuracy: 0.8000
    Epoch 8/10
    320/320 [==============================] - 0s 191us/sample - loss: 0.7212 - accuracy: 0.8156 - val_loss: 0.7024 - val_accuracy: 0.8000
    Epoch 9/10
    320/320 [==============================] - 0s 193us/sample - loss: 0.7305 - accuracy: 0.7719 - val_loss: 0.7032 - val_accuracy: 0.8000
    Epoch 10/10
    320/320 [==============================] - 0s 191us/sample - loss: 0.7183 - accuracy: 0.8188 - val_loss: 0.7020 - val_accuracy: 0.8000



```python
class Learning_Rate:
    def __init__(self, state_space, control_space):
        self.counter = dict(
                zip(list(product(state_space, control_space)), [0]*len(state_space)*len(control_space))
        )
    def __call__(self, i, s, a):
        self.counter[(s, a)] += 1
        return 1/self.counter[(s, a)]
# lr_fn = Learning_Rate(state_space, control_space)
# lr_fn = lambda i,s,a: 1/(i+1)
def model_wrapper(model):
    def m(s, x, acts):
        inputs = np.array([[x, acts[0]]])
        return model.predict(inputs)[0, int(s)]
    return m
lr_fn = lambda i,s,a: 0.1
env = delayed_wrapper(Env(), 1)
q = Q_table(random = False)
q_learn = Q_learning(q, env, 1, model, model_wrapper, MAX_ITER = 1000)
learned_q = q_learn.train(lr_fn)
```


```python
m = model_wrapper(model)
m(0, 1, [1])
```




    0.35112822




```python
v_l, pi_l= q_learn.q.yield_v_pi()
```


```python
v_l
```




    0    41.123725
    1    42.282631
    2    50.209974
    dtype: float64




```python
pi_l
```




    0    1
    1    1
    2   -1
    dtype: int64




```python
def eval_pi_model(env, q, model, model_wrapper, discount_factor, MAX_ITER = 1000000):

    path_len = int(np.log(1e-3)/np.log(discount_factor))
    path_num = int(MAX_ITER/path_len)
    obs = 0
    disc_arr = np.array([discount_factor**n for n in range(path_len)])
    r_ls = np.array([[np.nan]*path_len for _ in range(path_num)])
    ret = []
    delay_step = env.num
    action_buffer = [-1]*delay_step
    model = model_wrapper(model)
    for k in range(path_num):
        obs = env.reset(random= True)
        for i in range(path_len):
            action = get_opt_act(q, model, obs, action_buffer)
            action_buffer.pop(0)
            action_buffer.append(action)
            next_obs, reward, _, _ = env.step(action)
    #         discounted_cum_r = reward + discount_factor*discounted_cum_r
            r_ls[k, i] = reward
            obs = next_obs
        ret.append(np.cumsum(disc_arr*r_ls[k, :]))
    return np.mean(ret) ,ret
```


```python
val, v_samples_lin = eval_pi_model(env, q_learn.q, model, model_wrapper, 0.95)
val
```


```python
sns.distplot(v_samples_lin, label = 'model')
plt.legend()
```


```python

```


```python

```


```python

```


```python
df = np.arange(0, 100).reshape(25, -1)
df
```




    array([[ 0,  1,  2,  3],
           [ 4,  5,  6,  7],
           [ 8,  9, 10, 11],
           [12, 13, 14, 15],
           [16, 17, 18, 19],
           [20, 21, 22, 23],
           [24, 25, 26, 27],
           [28, 29, 30, 31],
           [32, 33, 34, 35],
           [36, 37, 38, 39],
           [40, 41, 42, 43],
           [44, 45, 46, 47],
           [48, 49, 50, 51],
           [52, 53, 54, 55],
           [56, 57, 58, 59],
           [60, 61, 62, 63],
           [64, 65, 66, 67],
           [68, 69, 70, 71],
           [72, 73, 74, 75],
           [76, 77, 78, 79],
           [80, 81, 82, 83],
           [84, 85, 86, 87],
           [88, 89, 90, 91],
           [92, 93, 94, 95],
           [96, 97, 98, 99]])




```python
# T = 3
T = 3
[N, D] = df.shape
dataX = np.zeros((N - T + 1, T, D))
for i in range(T, N + 1):
    dataX[i - T] = df[i - T:i, :]
    
dataX

# unfolded LSTM
```




    array([[[ 0.,  1.,  2.,  3.],
            [ 4.,  5.,  6.,  7.],
            [ 8.,  9., 10., 11.]],
    
           [[ 4.,  5.,  6.,  7.],
            [ 8.,  9., 10., 11.],
            [12., 13., 14., 15.]],
    
           [[ 8.,  9., 10., 11.],
            [12., 13., 14., 15.],
            [16., 17., 18., 19.]],
    
           [[12., 13., 14., 15.],
            [16., 17., 18., 19.],
            [20., 21., 22., 23.]],
    
           [[16., 17., 18., 19.],
            [20., 21., 22., 23.],
            [24., 25., 26., 27.]],
    
           [[20., 21., 22., 23.],
            [24., 25., 26., 27.],
            [28., 29., 30., 31.]],
    
           [[24., 25., 26., 27.],
            [28., 29., 30., 31.],
            [32., 33., 34., 35.]],
    
           [[28., 29., 30., 31.],
            [32., 33., 34., 35.],
            [36., 37., 38., 39.]],
    
           [[32., 33., 34., 35.],
            [36., 37., 38., 39.],
            [40., 41., 42., 43.]],
    
           [[36., 37., 38., 39.],
            [40., 41., 42., 43.],
            [44., 45., 46., 47.]],
    
           [[40., 41., 42., 43.],
            [44., 45., 46., 47.],
            [48., 49., 50., 51.]],
    
           [[44., 45., 46., 47.],
            [48., 49., 50., 51.],
            [52., 53., 54., 55.]],
    
           [[48., 49., 50., 51.],
            [52., 53., 54., 55.],
            [56., 57., 58., 59.]],
    
           [[52., 53., 54., 55.],
            [56., 57., 58., 59.],
            [60., 61., 62., 63.]],
    
           [[56., 57., 58., 59.],
            [60., 61., 62., 63.],
            [64., 65., 66., 67.]],
    
           [[60., 61., 62., 63.],
            [64., 65., 66., 67.],
            [68., 69., 70., 71.]],
    
           [[64., 65., 66., 67.],
            [68., 69., 70., 71.],
            [72., 73., 74., 75.]],
    
           [[68., 69., 70., 71.],
            [72., 73., 74., 75.],
            [76., 77., 78., 79.]],
    
           [[72., 73., 74., 75.],
            [76., 77., 78., 79.],
            [80., 81., 82., 83.]],
    
           [[76., 77., 78., 79.],
            [80., 81., 82., 83.],
            [84., 85., 86., 87.]],
    
           [[80., 81., 82., 83.],
            [84., 85., 86., 87.],
            [88., 89., 90., 91.]],
    
           [[84., 85., 86., 87.],
            [88., 89., 90., 91.],
            [92., 93., 94., 95.]],
    
           [[88., 89., 90., 91.],
            [92., 93., 94., 95.],
            [96., 97., 98., 99.]]])




```python
dataX.shape
```




    (23, 3, 4)




```python
[np.random.choice(range(24), replace = True) for _ in range(10)]
```




    [2, 6, 23, 17, 22, 11, 1, 13, 22, 14]




```python
dataX.reshape(dataX.shape + (1,)).shape
```




    (23, 3, 4, 1)




```python
time = range(1,6)
disc = np.array([np.exp(-0.05*t) for t in time])
disc
```




    array([0.95122942, 0.90483742, 0.86070798, 0.81873075, 0.77880078])




```python
def A(lamb):
    return 0.016*(np.array([np.exp(-n*lamb) for n in time])*disc).sum()

def B(lamb):
    return 0.5*0.016*C(lamb)

def C(lamb):
    return (disc*np.array([np.exp(-(n-1)*lamb)-np.exp(-n*lamb) for n in time])).sum()
```


```python
lamb = 0.0154
A(lamb)+B(lamb), C(lamb)
```




    (0.06654174415562401, 0.06404516300888626)




```python

```

753bb27d2ea3f0e148f0067dbbd85828918142a7
ipynb
Jupyter Notebook
W-maze_determ.ipynb
Theling/Delayed_MDP
db1a8808a05917a5915220947cf65268f0524fa9
MIT
W-maze_determ.ipynb
Theling/Delayed_MDP
db1a8808a05917a5915220947cf65268f0524fa9
MIT
W-maze_determ.ipynb
Theling/Delayed_MDP
db1a8808a05917a5915220947cf65268f0524fa9
MIT
Qwen/Qwen-72B
1. YES
2. YES