<filename>pydota2/ml_algo/q_learn.py
# NOTE: This code is to a large degree based on DeepMind work for 
#       AI in StarCraft2, just ported towards the Dota 2 game.
#       DeepMind's License is posted below.

# Copyright 2017 Google Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implement Q-Learning algorithm for use by agents."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import pandas as pd

# Based on https://github.com/MorvanZhou/Reinforcement-learning-with-tensorflow
class QLearning:
    def __init__(self, actions, learning_rate=0.1, reward_decay=0.0, e_greedy=0.95):
        self.actions = actions  # a list
        self.lr = learning_rate
        self.gamma = reward_decay
        self.epsilon = e_greedy
        self.q_table = pd.DataFrame(columns=self.actions)

    def choose_action(self, observation):
        self.check_state_exist(observation)
        
        if np.random.uniform() < self.epsilon:
            # choose best action
            state_action = self.q_table.ix[observation, :]
            
            # some actions have the same value
            state_action = state_action.reindex(np.random.permutation(state_action.index))
            
            action = state_action.argmax()
            #print("Best Action: ", str(action))
        else:
            # choose random action
            action = np.random.choice(self.actions)
            #print("Random Action: ", str(action))
            
        return action

    def learn(self, s, a, r, s_):
        self.check_state_exist(s_)
        self.check_state_exist(s)
        
        q_predict = self.q_table.ix[s, a]
        q_target = r + self.gamma * self.q_table.ix[s_, :].max()
        
        # update
        self.q_table.ix[s, a] += self.lr * (q_target - q_predict)

    def check_state_exist(self, state):
        if state not in self.q_table.index:
            # append new state to q table
            self.q_table = self.q_table.append(pd.Series([0] * len(self.actions), index=self.q_table.columns, name=state))
            
    def load_table(self, infile):
        with open(infile, 'r') as f:
            json_data = json.load(f)
            self.q_table = pd.io.json.json_normalize(json_data)
    
    def dump_table(self, outfile=None):
        if outfile:
            temp = sys.stdout
            sys.stdout = open('log.txt', 'w')
        
        print(self.q_table.to_json(orient='table'))
        
        if outfile:
            sys.stdout = temp

<filename>pydota2/ml_algo/q_learn.py
# NOTE: This code is to a large degree based on DeepMind work for 
#       AI in StarCraft2, just ported towards the Dota 2 game.
#       DeepMind's License is posted below.

# Copyright 2017 Google Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implement Q-Learning algorithm for use by agents."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import pandas as pd

# Based on https://github.com/MorvanZhou/Reinforcement-learning-with-tensorflow
class QLearning:
    def __init__(self, actions, learning_rate=0.1, reward_decay=0.0, e_greedy=0.95):
        self.actions = actions  # a list
        self.lr = learning_rate
        self.gamma = reward_decay
        self.epsilon = e_greedy
        self.q_table = pd.DataFrame(columns=self.actions)

    def choose_action(self, observation):
        self.check_state_exist(observation)
        
        if np.random.uniform() < self.epsilon:
            # choose best action
            state_action = self.q_table.ix[observation, :]
            
            # some actions have the same value
            state_action = state_action.reindex(np.random.permutation(state_action.index))
            
            action = state_action.argmax()
            #print("Best Action: ", str(action))
        else:
            # choose random action
            action = np.random.choice(self.actions)
            #print("Random Action: ", str(action))
            
        return action

    def learn(self, s, a, r, s_):
        self.check_state_exist(s_)
        self.check_state_exist(s)
        
        q_predict = self.q_table.ix[s, a]
        q_target = r + self.gamma * self.q_table.ix[s_, :].max()
        
        # update
        self.q_table.ix[s, a] += self.lr * (q_target - q_predict)

    def check_state_exist(self, state):
        if state not in self.q_table.index:
            # append new state to q table
            self.q_table = self.q_table.append(pd.Series([0] * len(self.actions), index=self.q_table.columns, name=state))
            
    def load_table(self, infile):
        with open(infile, 'r') as f:
            json_data = json.load(f)
            self.q_table = pd.io.json.json_normalize(json_data)
    
    def dump_table(self, outfile=None):
        if outfile:
            temp = sys.stdout
            sys.stdout = open('log.txt', 'w')
        
        print(self.q_table.to_json(orient='table'))
        
        if outfile:
            sys.stdout = temp

# NOTE: This code is to a large degree based on DeepMind work for #       AI in StarCraft2, just ported towards the Dota 2 game. #       DeepMind's License is posted below. # Copyright 2017 Google Inc. All Rights Reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #      http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS-IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. Implement Q-Learning algorithm for use by agents. # Based on https://github.com/MorvanZhou/Reinforcement-learning-with-tensorflow # a list # choose best action # some actions have the same value #print("Best Action: ", str(action)) # choose random action #print("Random Action: ", str(action)) # update # append new state to q table