title: Trouble with numpy arrays and matrices while doing backpropagation and gradient descent
body: <p>I'm following <a href="https://www.youtube.com/playlist?list=PLRqwX-V7Uu6aCibgK1PTWWu9by6XFdCfh" rel="nofollow noreferrer">this</a> video tutorial series by Dan Shiffman about creating a small "toy" neural network library.</p>
  
  <p>The tutorial uses JS and a matrix library he teaches how to code, earlier in the series. I, however, use numpy.</p>
  
  <p>In <a href="https://www.youtube.com/watch?v=qB2nwJxNVxM" rel="nofollow noreferrer">this</a> video he programs gradient descent and backpropagation. However, because I'm using numpy, my code doesn't seem too work. If anyone can help I'd be super thankful!</p>
  
  <p>Here is my code:</p>
  
  <pre><code>import numpy as np
  import math
  
  def sigmoid(x):
      return 1 / (1 + np.exp(-x))
  
  def dsigmoid(x):
      return x * (1 - x)
  
  class NeuralNetwork:
      def __init__(self, Inum, Hnum, Onum):
          self.Inum = Inum
          self.Hnum = Hnum
          self.Onum = Onum
  
          self.lr = 0.1
  
          self.weightsIH = np.random.rand(self.Hnum, self.Inum) * 2 - 1
          self.weightsHO = np.random.rand(self.Onum, self.Hnum) * 2 - 1
  
          self.biasH = np.random.rand(self.Hnum) * 2 - 1
          self.biasO = np.random.rand(self.Onum) * 2 - 1
  
      def feedForward(self, inputs):
          hidden = np.dot(self.weightsIH, np.array(inputs))
          hidden = hidden + self.biasH
          hidden = sigmoid(hidden)
  
          outputs = np.dot(self.weightsHO, hidden)
          outputs = outputs + self.biasO
          outputs = sigmoid(outputs)
  
          return outputs
  
      def train(self, inputs, targets):
          # Feed Forward
          hidden = np.dot(self.weightsIH, np.array(inputs))
          hidden = hidden + self.biasH
          hidden = sigmoid(hidden)
  
          outputs = np.dot(self.weightsHO, hidden)
          outputs = outputs + self.biasO
          outputs = sigmoid(outputs)
  
          # Calculate errors
          errorsO = np.array(targets) - outputs
  
          # Calculate gradients with derivitive of sigmoid
          # TODO: Use numpy for gradient calculation (if possible)
          gradients = dsigmoid(outputs)
          gradients = gradients * errorsO
          gradients = gradients * self.lr
  
          # Calculate deltas
          hiddenT = hidden[np.newaxis]
          weightsHODeltas = np.dot(gradients, hiddenT)
  
          # Adjust weights by deltas
          self.weightsHO = self.weightsHO + weightsHODeltas
  
          # Adjust bias by gradients
          self.biasO = self.biasO + gradients
  
  
          errorsH = np.transpose(self.weightsHO) * errorsO
  
  
          # Calculate gradients with derivitive of sigmoid
          # TODO: Use numpy for gradient calculation (if possible)
          gradientsH = dsigmoid(hidden)
          gradientsH = gradientsH * errorsH
          gradientsH = gradientsH * self.lr
  
          # Calculate deltas
          inputsT = np.array(inputs)[np.newaxis]
          weightsIHDeltas = np.dot(gradientsH, inputsT)
  
          # Adjust weights by deltas
          self.weightsIH = self.weightsIH + weightsIHDeltas
  
          # Adjust bias by gradients
          self.biasO = self.biasO + gradientsH
  </code></pre>
  
  <p>Here is the code I'm running:</p>
  
  <pre><code>from NN import NeuralNetwork
  from random import shuffle
  
  def main():
      nn = NeuralNetwork(2, 2, 1)
  
      dataset = [
          {
              "inputs": [0, 0],
              "outputs": 0
          },
          {
              "inputs": [0, 1],
              "outputs": 1
          },
          {
              "inputs": [1, 0],
              "outputs": 1
          },
          {
              "inputs": [1, 1],
              "outputs": 0
          }
      ]
  
  
      for x in range(100):
          for data in dataset:
              print(data)
              nn.train(data["inputs"], data["outputs"])
          shuffle(dataset)
  
      for data in dataset:
          print(data)
          nn.feedForward(data["inputs"])
  
  if __name__ == '__main__':
      main()
  </code></pre>
  
  <p>and here is the error message I'm getting:</p>
  
  <pre><code>Traceback (most recent call last):
  File "c:\Users\ghost\Desktop\Notes\Programming\Machine Learning\NN From Scratch\Yet Another Neural Network Library\main.py", line 38, in &lt;module&gt;
  main()
  File "c:\Users\ghost\Desktop\Notes\Programming\Machine Learning\NN From Scratch\Yet Another Neural Network Library\main.py", line 30, in main
  nn.train(data["inputs"], data["outputs"])
  File "c:\Users\ghost\Desktop\Notes\Programming\Machine Learning\NN From Scratch\Yet Another Neural Network Library\NN.py", line 77, in train
  weightsIHDeltas = np.dot(gradientsH, inputsT)
  ValueError: shapes (2,2) and (1,2) not aligned: 2 (dim 1) != 1 (dim 0)
  </code></pre>