title: Neural network only learns the last pattern when given several patterns sequentially
body: <p><a href="https://stackoverflow.com/questions/20806156/multiple-inputs-for-backpropagation-neural-network">This post</a> is about the same issue, but no proper answer has been given. And since this problem seems to be widespread, I'll keep my code behind the scene.</p>
  <p>Following <a href="http://neuralnetworksanddeeplearning.com/chap2.html" rel="nofollow noreferrer">this source</a>, I've written a network which does well when I give it a training example with a target vector. Using gradient descent I minimize the cost function to make the network provide the target vector when given the corresponding input vector. <strong>But this only works for one example!</strong></p>
  <p>The main goal of a neural network is to react differently for different inputs, and we should be able to train it to do so. I tried changing network weights by an average of delta-weights computed for each example, which failed: the training process gets stuck with the output vector holding averages of all the target vectors from the training set. No ideas left, no sources found to explain.</p>
  <p>How do I train a neural network with a set of examples, not with just one input vector?</p>
  <p><strong>Update</strong>
  For those wondering I'll attach my code below. Try to run this and you will see that instead of outputting <code>0 1</code> it provides <code>0.5 0.5</code>, which is the result of subtracting averaged delta-weights.</p>
  <pre><code>import numpy as np
  from sympy import symbols, lambdify
  from sympy.functions.elementary.exponential import exp
  from time import sleep
  
  x = symbols('x')
  sigmoid = exp(x) / (1 + exp(x))
  sigmoid_der = sigmoid.diff(x)
  sigmoid = lambdify(x, sigmoid)
  sigmoid_der = lambdify(x, sigmoid_der)
  
  
  class Neuron:
      def __init__(self, amount_of_inputs: int, hidden = True):
          self.inputs = np.random.rand(amount_of_inputs) if hidden else np.array([1])
          self.bias = 0.0
          self._activation = 0.0
          self._wsum = 0.0
  
      
      @property
      def activation(self) -&gt; float:
          return self._activation
  
      @property
      def wsum(self) -&gt; float:
          return self._wsum
  
  
      def calculate(self, indata):
          wval = self.inputs * indata + self.bias
          self._wsum = wval.sum()
          self._activation = sigmoid(self._wsum)
  
  
  
  class NeuralNetwork:
      def __init__(self, shape: tuple):
          self.shape = shape
          self.layers = len(self.shape)
  
          self.network = [None for _ in range(self.layers)]
          self.network[0] = tuple([Neuron(1, hidden = False) for _ in range(shape[0])])
          for L in range(1, self.layers):
              self.network[L] = tuple([Neuron(shape[L - 1]) for _ in range(shape[L])])
          self.network = tuple(self.network)
          
          y = [symbols(f'y[{i}]') for i in range(shape[self.layers - 1])]
          a = [symbols(f'a[{i}]') for i in range(shape[self.layers - 1])]
          self.cost_function = sum([(y[i] - a[i]) ** 2 / 2 for i in range(shape[self.layers - 1])])
          self.gradient = tuple([self.cost_function.diff(a[i]) for i in range(shape[self.layers - 1])])
          self.cost_function = lambdify((y, a), self.cost_function)
          self.gradient = lambdify((y, a), self.gradient)
  
  
      def getLayer(self, L):
          return np.array([self.network[L][i].activation for i in range(self.shape[L])])
  
      
      def getWeightedSum(self, L):
          return np.array([self.network[L][i].wsum for i in range(self.shape[L])])
  
      
      def getInputsMatrix(self, L):
          return np.array([self.network[L][i].inputs for i in range(self.shape[L])])
  
      
      def calculate(self, values):
          for i in range(self.shape[0]):
              self.network[0][i].calculate(values[i])
          
          for L in range(1, self.layers):
              indata = self.getLayer(L - 1)
              for j in range(self.shape[L]):
                  self.network[L][j].calculate(indata)
  
      
      def get_result(self) -&gt; tuple:
          return tuple([self.network[self.layers - 1][i].activation for i in range(self.shape[self.layers - 1])])
      
  
      def teach(self, targets, examples):
          if len(targets) != len(examples):
              raise TypeError(&quot;The amounts of target and input vectors do not coincide&quot;)
          
          activations = [None for _ in range(len(examples))]
          delta = activations.copy()
  
          cost_is_low_enough = False
          while not cost_is_low_enough:
              for x in range(len(examples)):
                  self.calculate(examples[x])
  
                  activations[x] = [self.getLayer(l) for l in range(self.layers)]
                  delta[x] = [None for _ in range(self.layers - 1)]
  
                  network_output = self.getLayer(self.layers - 1)
                  output_weighted = self.getWeightedSum(self.layers - 1)
                  gradient_vector = np.array(self.gradient(targets[x], network_output))
                  delta[x][-1] = gradient_vector * sigmoid_der(output_weighted)
  
                  for l in range(self.layers - 2, 0, -1):
                      weight_matrix = self.getInputsMatrix(l + 1).transpose()
                      output_weighted = self.getWeightedSum(l)
                      activation = self.getLayer(l)
                      for j in range(self.shape[l]):
                          delta[x][l - 1] = (weight_matrix @ delta[x][l]) * sigmoid_der(output_weighted) * activation
  
                 
              dw = [None for _ in range(self.layers - 1)]
              for x in range(len(examples)):
                  self.calculate(examples[x])
                  for l in range(self.layers - 1):
                      dw[l] = np.empty(self.shape[l + 1])
                      for j in range(self.shape[l + 1]):
                          dw[l][j] = np.mean([delta[x][l][j] for x in range(len(examples))])
  
              for l in range(1, self.layers):
                  for j in range(self.shape[l]):
                      for k in range(self.shape[l - 1]):
                          self.network[l][j].inputs[k] -= 0.1 * dw[l - 1][j]
              cost = 0
              for x in range(len(examples)):
                  self.calculate(examples[x])
                  network_output = np.array(self.get_result())
                  incost = self.cost_function(targets[x], network_output)
                  print(network_output, incost)
                  cost += incost
                  # sleep(0.05)
              cost /= len(examples)
              print()
              if cost &lt; 0.001: cost_is_low_enough = True
  
  
  
  network = NeuralNetwork((2, 4, 1))
  
  examples = np.array([
      [1, 2],
      [3, 4],
  ])
  
  targets = np.array([
      [0],
      [1]
  ])
  
  
  network.teach(targets, examples)
  
  values_1 = np.array([5, 10])
  network.calculate(values_1)
  result = network.get_result()
  print(result)
  
  '''
  values_2 = np.array([3, 4])
  network.calculate(values_2)
  result = network.get_result()
  print(result)
  '''
  </code></pre>