title: Neural net decision boundaries are perpendicular to true boundaries
body: <p>I have built a small neural net taking in two inputs. Two neurons in hidden layer, one neuron in output layer. The results are "mirror images" ie the decisions boundaries are perpendicular to the true boundaries. Does anyone know how this might be happening on what mistake I may have made?</p>
  
  <p><a href="https://i.stack.imgur.com/Nzanf.png" rel="nofollow noreferrer">linear data</a></p>
  
  <p><a href="https://i.stack.imgur.com/KA8Zs.png" rel="nofollow noreferrer">nonlinear data</a></p>
  
  <pre><code>import random, numpy, math
  
  
  lr = 0.1 #learning rate
  dt = '4' #data type 1: linear 2: curve 3: box 4: XORish
  epochs = 100000
  tda = 50 #training data amount
  
  
  def step(x): #step function
    if x &gt; 0:
      x = 1
    else:
      x = 0
    return x
  
  def error(truth, output):
    return 0.5 * (truth - output)**2
  
  def sig(x): #sigmoid activation
    return 1/(1+numpy.exp(-x))
  
  
  #weights
  w = [random.random(),random.random(),random.random(),random.random(),random.random(),random.random()]
  
  
  
  #biases
  b = [random.random(),random.random(),random.random()]
  
  
  def Net(x, y, t) : # t is truth (or target)
  
  
    h1 = x*w[0]+y*w[1]+b[0] #summation in h1, first neuron in hidden layer
    h1out = sig(h1) #sigmoid activation
  
  
    h2 = x*w[2]+y*w[3]+b[1]
    h2out = sig(h2) 
  
  
    z = h1out*w[4]+h2out*w[5]+b[2] #z is output neuron
    zout = sig(z)
  
    e = error(t, zout) # e is error
  
  
  
    #backpropagation, partial differentiations to find error at each weight and bias
    e5 = (zout-t) * (zout * (1 - zout)) * h1out #e5 is error at weight 5 etc
    e6 = (zout-t) * (zout * (1 - zout)) * h2out
  
    e1 = (zout-t) * (zout * (1 - zout)) * w[4] * (h1out * (1 - h1out)) * x
    e2 = (zout-t) * (zout * (1 - zout)) * w[4] * (h1out * (1 - h1out)) * y
    e3 = (zout-t) * (zout * (1 - zout)) * w[5] * (h2out * (1 - h2out)) * x
    e4 = (zout-t) * (zout * (1 - zout)) * w[5] * (h2out * (1 - h2out)) * y
  
  
    be3 = (zout-t) * (zout * (1 - zout))
    be1 = (zout-t) * (zout * (1 - zout)) * w[4] * (h1out * (1 - h1out)) 
    be2 = (zout-t) * (zout * (1 - zout)) * w[5] * (h2out * (1 - h2out)) 
  
  
  
    #updating weights and biases
    w[0] = w[0] - (e1 * lr)
    w[1] = w[1] - (e2 * lr)
    w[2] = w[2] - (e3 * lr)
    w[3] = w[3] - (e4 * lr)
  
    w[4] = w[4] - (e5 * lr)
    w[5] = w[5] - (e6 * lr)
  
  
    b[2] = b[2] - (be3 * lr)
  
    b[0] = b[0] - (be1 * lr)
    b[1] = b[1] - (be2 * lr)
  
  
  
  train_data = []
  
  while len(train_data)&lt;tda: #makes training data
    x = random.randrange(100)
    y = random.randrange(100)
    if dt == '1':
      if x+y&gt;100:
        truth = 1
      else:
        truth = 0
    elif dt == '2':
      if x*y&gt;1000:
        truth = 1
      else:
        truth = 0
    elif dt == '3':
      if x&gt;50 or y&gt;50:
        truth = 1
      else:
        truth = 0
    elif dt == '4':
      if x+y&gt;60 and x+y&lt;140:
        truth = 1
      else:
        truth = 0
    if [x,y,truth] not in train_data:
      train_data.append([x,y,truth])
  
  
  
  for n in range(epochs): #EPOCHS
    for i in train_data:
      Net(i[0], i[1], i[2])
    if n%1000 == 0 and n != 0:
      print (n, 'epochs')
  
    if n%5000 == 0:
  
  #every 5000 epochs the net is fed new test data and results are plotted 
  
      test_data = []
  
      while len(test_data)&lt;(50): #makes test data
        x = random.randrange(100)
        y = random.randrange(100)
        if [x, y] not in test_data and [x, y, 0] not in train_data and [x, y, 1] not in train_data:
          test_data.append([x, y])
  
  
  #classifying the test data with the net
      for i in test_data:
        x = i[0]
        y = i[1]
  
  
        h1 = x*w[0]+y*w[1]+b[0]
        h1out = sig(h1)
  
  
        h2 = x*w[2]+y*w[3]+b[1]
        h2out = sig(h2)
  
  
        z = h1out*w[4]+h2out*w[5]+b[2]
        zout = step(z)
  
  
        i.append(zout)
  
  
      print (test_data)
      print (w, b)
  
  #plotting results
      import matplotlib.pyplot as plt
  
      x0 = []
      y0 = []
      x1 = []
      y1 = []
      xt0 = []
      yt0 = []
      xt1 = []
      yt1 = []
  
      for i in train_data:
        if i[2] == 0:
          x0.append(i[0])
          y0.append(i[1])
        else:
          x1.append(i[0])
          y1.append(i[1])
  
      for i in test_data:
        if i[2] == 0:
          xt0.append(i[0])
          yt0.append(i[1])
        else:
          xt1.append(i[0])
          yt1.append(i[1])
  
      plt.clf()
      plt.scatter(xt0, yt0, 30, color = 'red')
      plt.scatter(xt1, yt1, 30, color = 'blue')
      plt.scatter(x0, y0, 10, color = 'orange')
      plt.scatter(x1, y1, 10, color = 'green')
      plt.title(str(tda)+" training data, "+str(epochs)+" epochs \n Red/Orange are small things, Blue/Green are big things \n Orange/Green are training data, Red/Blue are test data")
      plt.xlabel("Width")
      plt.ylabel("Height")
      plt.savefig('plot.png')
  </code></pre>