title: Suggestions for non-working Radial Basis Function Neural Network
body: <p>I'm creating a small C# application to help investigate different designs of Multilayer Perceptron and Radial Basis Function Neural Networks. The MLP is working adequately, but I can't manage to get the RBF Net to work at all.
  I've checked and double checked and triple checked the algorithms to see if they match the algorithms that are available online and in papers and books and it seems like they do. I've also checked a colleagues (working) code and compared it to mine to see if there is anything I'd done wrong or left out and found nothing. 
  So I was hoping some extra eyes and opinions might help me root out the problem, as I've run out of ideas of what to try or where to look. If you want to have a look at or download the entire project you can find it at <a href="https://bitbucket.org/floofykh/gameai-coursework" rel="nofollow">https://bitbucket.org/floofykh/gameai-coursework</a></p>
  
  <p>I'll also post here the RBF specific code. </p>
  
  <p>This is the Windows Form which allows the user to input the design of the RBF net and train it. </p>
  
  <pre><code>public partial class RBFForm : Form
  {
      private const double X_MAX = Math.PI * 2;
      private const double X_MIN = 0;
      private const double INTERVAL = Math.PI / 90d;
      private double m_numPlotPoints;
      private double m_noiseValue = 0;
      public double StopThreshold { get { return m_stopThreshold; } }
      private double m_stopThreshold;
      private string m_function = "";
      private List&lt;double&gt; m_inputs, m_targets;
      private List&lt;RadialBasisFunctionData&gt; m_rbfsData;
      private RadialBasisFunctionNetwork m_rbf;
      private int m_numRBFs = 1;
      private double m_rbfWidth = 1d, m_rbfOffset = 0d, m_rbfSeperation = 0d;
      private bool m_changed = true;
      private const int testCases = 180;
  
      public RBFForm()
      {
          InitializeComponent();
          ChartArea functionArea = m_functionGraph.ChartAreas.Add("function");
          functionArea.AxisX.Maximum = X_MAX;
          functionArea.AxisX.Minimum = X_MIN;
          functionArea.AxisY.Maximum = 1.5;
          functionArea.AxisY.Minimum = -1.5;
          ChartArea rbfArea = m_rbfGraph.ChartAreas.Add("RBFs");
          rbfArea.AxisX.Maximum = X_MAX;
          rbfArea.AxisX.Minimum = X_MIN;
          rbfArea.AxisY.Maximum = 1;
          rbfArea.AxisY.Minimum = 0;
  
          m_functionGraph.Series.Add("Neural Network");
          m_functionGraph.Series.Add("Function");
          m_functionGraph.Series.Add("Points");
          m_rbfGraph.Series.Add("RBFs");
  
          Neuron.LearningRate = ((double)(m_learningRateSelector).Value);
          m_numRBFs = ((int)(m_numRBFsInput).Value);
          m_rbfOffset = ((double)(m_rbfOffsetController).Value);
          m_rbfSeperation = ((double)(m_rbfOffsetController).Value);
          m_rbfWidth = ((double)(m_rbfWidthController).Value);
  
          m_rbf = new RadialBasisFunctionNetwork(this);
      }
      private void InitialiseFunctionGraph()
      {
          Series func = m_functionGraph.Series.FindByName("Function");
          func.Points.Clear();
          func.ChartType = SeriesChartType.Line;
          func.Color = Color.Green;
          func.BorderWidth = 1;
  
          for (double x = X_MIN; x &lt; X_MAX; x += INTERVAL)
          {
              double y = 0;
              switch (m_function)
              {
                  case "Sin":
                      y = Math.Sin(x);
                      break;
                  case "Cos":
                      y = Math.Cos(x);
                      break;
              };
              func.Points.AddXY(x, y);
          }
      }
  
      private void InitialiseRBFs()
      {
          m_rbfsData = new List&lt;RadialBasisFunctionData&gt;();
          Series rbfs = m_rbfGraph.Series.FindByName("RBFs");
          rbfs.Points.Clear();
          rbfs.ChartType = SeriesChartType.Line;
          rbfs.Color = Color.IndianRed;
          rbfs.BorderWidth = 1;
  
          for(int i=0; i&lt;m_numRBFs; i++)
          {
              double centre = X_MIN + m_rbfOffset + m_rbfSeperation * i;
              RadialBasisFunctionData data = new RadialBasisFunctionData();
              data.Centre = centre;
              data.Width = m_rbfWidth;
              m_rbfsData.Add(data);
              DrawRBF(centre, m_rbfWidth, rbfs.Points);
          }
      }
  
      private void DrawRBF(double centre, double width, DataPointCollection points)
      {
          if(width &gt; 0)
          {
              IActivationFunction function = new RadialBasisFunction(centre, width);
              for (double x = X_MIN; x &lt; X_MAX; x += INTERVAL)
              {
                  double y = function.Function(x);
                  points.AddXY(x, y);
              }
          }
      }
  
      private void InitialiseInputPoints()
      {
          m_inputs = new List&lt;double&gt;();
          m_targets = new List&lt;double&gt;();
  
          Series points = m_functionGraph.Series.FindByName("Points");
          points.Points.Clear();
          points.ChartType = SeriesChartType.Point;
          points.Color = Color.Blue;
          points.BorderWidth = 1;
  
          double interval = 0d;
          if (m_numPlotPoints &gt; 1)
              interval = (X_MAX - X_MIN) / (m_numPlotPoints - 1);
  
          for (int point = 0; point &lt; m_numPlotPoints; point++)
          {
              double x = X_MIN + point * interval;
              double y = 0;
              switch (m_function)
              {
                  case "Sin":
                      y = Math.Sin(x);
                      break;
                  case "Cos":
                      y = Math.Cos(x);
                      break;
              };
  
              y += (Program.rand.NextDouble() - 0.5d) * 2d * m_noiseValue;
              m_targets.Add(y);
              m_inputs.Add(x);
              points.Points.AddXY(x, y);
          }
      }
      public void SetNumEpochs(int num)
      {
          m_numEpochLabel.Text = num.ToString();
      }
  
      public void SetNumEpochsAsync(int num)
      {
          try
          {
              if (m_numEpochLabel.InvokeRequired)
              {
                  m_numEpochLabel.Invoke((MethodInvoker)delegate
                  {
                      m_numEpochLabel.Text = num.ToString();
                  });
              }
          }
          catch (Exception) { }
      }
  
      private void m_rbfSeperationController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          m_rbfSeperation = value;
          InitialiseRBFs();
          m_changed = true;
      }
  
      private void m_numRBFsInput_ValueChanged(object sender, EventArgs e)
      {
          int value = ((int)((NumericUpDown)sender).Value);
          m_numRBFs = value;
          InitialiseRBFs();
          m_changed = true;
      }
  
      private void m_rbfWidthController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          m_rbfWidth = value;
          InitialiseRBFs();
          m_changed = true;
      }
  
      private void m_rbfOffsetController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          m_rbfOffset = value;
          InitialiseRBFs();
          m_changed = true;
      }
  
      private void m_learningRateSelector_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          Neuron.LearningRate = value;
          m_changed = true;
      }
  
      private void m_momentumController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          Neuron.MomentumAlpha = value;
          m_changed = true;
      }
  
      private void m_thresholdController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          m_stopThreshold = value;
          m_changed = true;
      }
  
      private void m_functionSelector_SelectedIndexChanged(object sender, EventArgs e)
      {
          m_function = ((ComboBox)sender).SelectedItem.ToString();
          InitialiseFunctionGraph();
          m_changed = true;
      }
  
      private void m_plotPointsController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          m_numPlotPoints = value;
          InitialiseInputPoints();
          m_changed = true;
      }
  
      private void m_noiseController_ValueChanged(object sender, EventArgs e)
      {
          double value = ((double)((NumericUpDown)sender).Value);
          m_noiseValue = value;
          InitialiseInputPoints();
          m_changed = true;
      }
  
      private void m_trainButton_Click(object sender, EventArgs e)
      {
          if (m_rbf != null)
          {
              if (RadialBasisFunctionNetwork.Running)
              {
                  RadialBasisFunctionNetwork.Running = false;
              }
              else
              {
                  if (m_changed)
                  {
                      m_rbf.Initialise(1, m_rbfsData, 1);
                      m_changed = false;
                  }
  
                  RadialBasisFunctionNetwork.ErrorStopThreshold = m_stopThreshold;
                  List&lt;List&lt;double&gt;&gt; inputPatterns = new List&lt;List&lt;double&gt;&gt;();
                  List&lt;List&lt;double&gt;&gt; targetPatterns = new List&lt;List&lt;double&gt;&gt;();
  
                  for (int i = 0; i &lt; m_inputs.Count; i++)
                  {
                      List&lt;double&gt; newInputPattern = new List&lt;double&gt;();
                      newInputPattern.Add(m_inputs[i]);
                      List&lt;double&gt; newTargetPattern = new List&lt;double&gt;();
                      newTargetPattern.Add(m_targets[i]);
  
                      inputPatterns.Add(newInputPattern);
                      targetPatterns.Add(newTargetPattern);
  
                  }
  
                  m_rbf.Train(inputPatterns, targetPatterns);
              }
          }
      }
  
      public void TestAndPresent()
      {
          List&lt;double&gt; finalData = new List&lt;double&gt;();
  
          for (double x = X_MIN; x &lt; X_MAX; x += INTERVAL)
          {
              List&lt;double&gt; input = new List&lt;double&gt;();
              input.Add(x);
  
              finalData.AddRange(m_rbf.Test(input));
          }
          PlotNeuralOutput(finalData);
      }
      public void TestAndPresentAsync()
      {
          List&lt;double&gt; finalData = new List&lt;double&gt;();
  
          for (double x = X_MIN; x &lt; X_MAX; x += INTERVAL)
          {
              List&lt;double&gt; input = new List&lt;double&gt;();
              input.Add(x);
  
              finalData.AddRange(m_rbf.Test(input));
          }
          PlotNeuralOutputAsync(finalData);
      }
      public void PlotNeuralOutput(List&lt;double&gt; output)
      {
          Series network = m_functionGraph.Series["Neural Network"];
  
          network.Points.Clear();
          network.ChartType = SeriesChartType.Line;
          network.Color = Color.Red;
          network.BorderWidth = 3;
  
          double x = 0;
          for (int i = 0; i &lt; output.Count; i++)
          {
              network.Points.AddXY(x, output[i]);
              x += INTERVAL;
          }
      }
      public void PlotNeuralOutputAsync(List&lt;double&gt; output)
      {
          try
          {
              if (m_functionGraph.InvokeRequired)
              {
                  m_functionGraph.Invoke((MethodInvoker)delegate
                  {
                      Series network = m_functionGraph.Series["Neural Network"];
  
                      network.Points.Clear();
                      network.ChartType = SeriesChartType.Line;
                      network.Color = Color.Red;
                      network.BorderWidth = 3;
  
                      double x = 0;
                      for (int i = 0; i &lt; output.Count; i++)
                      {
                          network.Points.AddXY(x, output[i]);
                          x += INTERVAL;
                      }
                  });
              }
          }
          catch (Exception) { }
      }
  }
  </code></pre>
  
  <p>Here is the RadialBasisFunction class where most of the RBF algorithm takes place, specificaly in FeedForward().</p>
  
  <pre><code>class RadialBasisFunctionNetwork
  {
  
      private NeuronLayer m_inputLayer;
      private NeuronLayer m_radialFunctions;
      private NeuronLayer m_outputLayer;
  
      private int m_numRadialFunctions = 0;
      public static bool Running = false;
  
      public static double ErrorStopThreshold {get; set;}
      private static int m_epoch = 0;
      public static int Epoch { get { return m_epoch; } }
  
      private RBFForm m_RBFForm = null;
  
      public RadialBasisFunctionNetwork(RBFForm RBFForm)
      {
          m_RBFForm = RBFForm;
          m_inputLayer = new NeuronLayer();
          m_radialFunctions = new NeuronLayer();
          m_outputLayer = new NeuronLayer();
      }
  
      public void Initialise(int numInputs, List&lt;RadialBasisFunctionData&gt; radialFunctions, int numOutputs)
      {
          ErrorStopThreshold = 0d;
          m_epoch = 0;
          m_numRadialFunctions = radialFunctions.Count;
  
          m_inputLayer.Neurons.Clear();
          //Add bias neuron
          /*Neuron inputBiasNeuron = new Neuron(1d);
          inputBiasNeuron.Initialise(m_numRadialFunctions);
          m_inputLayer.Neurons.Add(inputBiasNeuron);*/
          for(int i=0; i&lt;numInputs; i++)
          {
              Neuron newNeuron = new Neuron();
              newNeuron.Initialise(m_numRadialFunctions);
              m_inputLayer.Neurons.Add(newNeuron);
          }
  
          m_outputLayer.Neurons.Clear();
          for (int i = 0; i &lt; numOutputs; i++)
          {
              Neuron newNeuron = new Neuron();
              m_outputLayer.Neurons.Add(newNeuron);
          }
  
          m_radialFunctions.Neurons.Clear();
          //Add bias neuron
         /* Neuron outputBiasNeuron = new Neuron(1d);
          outputBiasNeuron.Initialise(numOutputs);
          outputBiasNeuron.ActivationFunction = new ConstantActivationFunction();
          m_radialFunctions.Neurons.Add(outputBiasNeuron);*/
          for (int i = 0; i &lt; m_numRadialFunctions; i++)
          {
              Neuron newNeuron = new Neuron();
              newNeuron.Initialise(numOutputs);
              newNeuron.ActivationFunction = new RadialBasisFunction(radialFunctions[i].Centre, radialFunctions[i].Width);
              m_radialFunctions.Neurons.Add(newNeuron);
          }
      }
  
      public void Train(List&lt;List&lt;double&gt;&gt; inputs, List&lt;List&lt;double&gt;&gt; targets)
      {
          Running = true;
          BackgroundWorker bw = new BackgroundWorker();
  
          bw.DoWork += new DoWorkEventHandler(
              delegate(object o, DoWorkEventArgs args)
              {
                  while (Running)
                  {
                      TrainPatterns(inputs, targets);
  
                      m_RBFForm.SetNumEpochsAsync(m_epoch);
                      m_RBFForm.TestAndPresentAsync();
                  }
              });
  
          bw.RunWorkerAsync();
      }
  
      private void TrainPatterns(List&lt;List&lt;double&gt;&gt; inputs, List&lt;List&lt;double&gt;&gt; targets)
      {
          Queue&lt;int&gt; randomIndices = GenRandomNonRepNumbers(inputs.Count, 0, inputs.Count, Program.rand);
  
          bool trained = true;
          while(randomIndices.Count &gt; 0)
          {
              int index = randomIndices.Dequeue();
              TrainPattern(inputs[index], targets[index]);
              foreach(Neuron neuron in m_outputLayer.Neurons)
              {
                  if (Math.Abs(neuron.Error) &gt; ErrorStopThreshold)
                      trained = false;
              }
          }
          m_epoch++;
  
          if (trained)
              Running = false;
      }
  
      public void TrainPattern(List&lt;double&gt; inputs, List&lt;double&gt; targets)
      {
          InitialisePatternSet(inputs, targets);
          FeedForward();
      }
  
      public List&lt;double&gt; Test(List&lt;double&gt; inputs)
      {
          InitialisePatternSet(inputs, null);
          FeedForward();
          return GetOutput();
      }
  
      public void FeedForward()
      {
          //Feed from input
          for(int i=0; i&lt;m_radialFunctions.NumNeurons; i++)
          {
              Neuron radialFunctionNeuron = m_radialFunctions.Neurons[i];
              radialFunctionNeuron.Output = 0d;
              for(int j=0; j&lt;m_inputLayer.NumNeurons; j++)
              {
                  Neuron inputNeuron = m_inputLayer.Neurons[j];
                  radialFunctionNeuron.Output += radialFunctionNeuron.ActivationFunction.Function(inputNeuron.Output);
              }
          }
  
          //Feed to output
          for (int i = 0; i &lt; m_outputLayer.NumNeurons; i++)
          {
              Neuron outputNeuron = m_outputLayer.Neurons[i];
              outputNeuron.Output = 0d;
              for (int j = 0; j &lt; m_radialFunctions.NumNeurons; j++)
              {
                  Neuron radialFunctionNeuron = m_radialFunctions.Neurons[j];
                  outputNeuron.Output += radialFunctionNeuron.Weight(i) * radialFunctionNeuron.Output;
              }
              outputNeuron.Error = (outputNeuron.Target - outputNeuron.Output);
          }
  
          //Update weights
          for (int i = 0; i &lt; m_radialFunctions.NumNeurons; i++)
          {
              Neuron radialFunctionNeuron = m_radialFunctions.Neurons[i];
              for (int j = 0; j &lt; m_outputLayer.NumNeurons; j++)
              {
                  Neuron outputNeuron = m_outputLayer.Neurons[j];
                  if(Math.Abs(outputNeuron.Error) &gt; m_RBFForm.StopThreshold)
                      radialFunctionNeuron.m_weights[j] += Neuron.LearningRate * outputNeuron.Error * radialFunctionNeuron.Output;
              }
          }
      }
  
      public List&lt;double&gt; GetOutput()
      {
          List&lt;double&gt; output = new List&lt;double&gt;();
          for (int i = 0; i &lt; m_outputLayer.NumNeurons; i++)
          {
              output.Add(m_outputLayer.Neurons[i].Output);
          }
          return output;
      }
  
      private void InitialisePatternSet(List&lt;double&gt; inputs, List&lt;double&gt; targets)
      {
          m_inputLayer.SetInputs(inputs, false);
  
          if(targets != null)
          {
              m_outputLayer.SetTargets(targets);
          }
      }
  
      private Queue&lt;int&gt; GenRandomNonRepNumbers(int num, int min, int max, Random generator)
      {
          if (max - min &lt; num)
              return null;
  
          Queue&lt;int&gt; numbers = new Queue&lt;int&gt;(num);
  
          for (int i = 0; i &lt; num; i++)
          {
              int randNum = 0;
              do
              {
                  randNum = generator.Next(min, max);
              } while (numbers.Contains(randNum));
  
              numbers.Enqueue(randNum);
          }
  
          return numbers;
      }
  }
  </code></pre>
  
  <p>This is the Radial Basis Function I am using as an activation function</p>
  
  <pre><code>class RadialBasisFunction : IActivationFunction
  {
      private double m_centre = 0d, m_width = 0d;
  
      public RadialBasisFunction(double centre, double width)
      {
          m_centre = centre;
          m_width = width;
      }
  
      double IActivationFunction.Function(double activation)
      {
          double dist = activation - m_centre;
          return Math.Exp(-(dist * dist) / (2 * m_width * m_width));
          //return Math.Exp(-Math.Pow(dist / (2 * m_width), 2d));
          //return Math.Exp(-Math.Pow(dist, 2d));
      }
  }
  </code></pre>
  
  <p>The NeuronLayer class is really just a wrapper around a List of Neurons, and isn't entirely necessary anymore, but I've been focussing on getting everything working rather than keeping my code clean and well designed. </p>
  
  <pre><code>class NeuronLayer
  {
      public int NumNeurons { get { return Neurons.Count; } }
      public List&lt;Neuron&gt; Neurons { get; set; }
  
      public NeuronLayer ()
      {
          Neurons = new List&lt;Neuron&gt;();
      }
  
      public void SetInputs(List&lt;double&gt; inputs, bool skipBias)
      {
          for (int i = 0; i &lt; Neurons.Count; i++)
          {
              if(skipBias)
              {
                  if (i != 0)
                      Neurons[i].Input = inputs[i-1];
              }
              else
              {
                  Neurons[i].Input = inputs[i];
              }
          }
      }
  
      public void SetTargets(List&lt;double&gt; targets)
      {
          for (int i = 0; i &lt; Neurons.Count; i++)
          {
              Neurons[i].Target = targets[i];
          }
      }
  }
  </code></pre>
  
  <p>And finally the Neuron class. This class was made while I was coding the MLP and while I was still trying to figure out exactly how Neural Nets work. So unfortunately a lot of the code in it is specific to MLPs. I hope to change this once I've got everything working and can start cleaning everything and make the application more user-friendly. I'm going to add all the functions for completeness, but I've checked and double checked and I shouldn't be using any of the MLP specific code of Neuron anywhere in my RBF network. The MLP specific stuff is WithinThreshold, and all the functions after Weight(int).</p>
  
  <pre><code>class Neuron
  {
      private IActivationFunction m_activationFunction = null;
      public IActivationFunction ActivationFunction { get { return m_activationFunction; } set { m_activationFunction = value; } }
  
      public double Input { get { return Output; } set { Output = value; } }
      public double Output { get; set; }
      public double Error { get; set; }
      public double Target { get; set; }
  
      private double m_activation = 0d;
  
      public bool WithinThreshold { get { return Math.Abs(Error) &lt; MultilayerPerceptron.ErrorStopThreshold; } }
  
      public static double LearningRate { get; set; }
      public static double MomentumAlpha { get; set; }
  
      public List&lt;double&gt; m_weights;
      private List&lt;double&gt; m_deltaWeights;
  
      public Neuron()
      {
          Output = 0d;
          m_weights = new List&lt;double&gt;();
          m_deltaWeights = new List&lt;double&gt;();
          m_activationFunction = new TanHActFunction();
      }
  
      public Neuron(double input)
      {
          Input = input;
          Output = input;
          m_weights = new List&lt;double&gt;();
          m_deltaWeights = new List&lt;double&gt;();
          m_activationFunction = new TanHActFunction();
      }
  
      public void Initialise(int numWeights)
      {
          for(int i=0; i&lt;numWeights; i++)
          {
              m_weights.Add(Program.rand.NextDouble()*2d - 1d);
          }
      }
  
      public double Weight(int index)
      {
          if (m_weights != null &amp;&amp; m_weights.Count &gt; index)
              return m_weights[index];
  
          return 0d;
      }
  
      public void Feed(NeuronLayer layer, int neuronIndex)
      {
          List&lt;Neuron&gt; inputNeurons = layer.Neurons;
          m_activation = 0;
  
          for (int j = 0; j &lt; layer.NumNeurons; j++)
          {
              m_activation += inputNeurons[j].Output * inputNeurons[j].Weight(neuronIndex);
          }
  
          Output = m_activationFunction.Function(m_activation);
      }
  
      public void CalculateError(NeuronLayer successor, bool outputLayer)
      {
          if(outputLayer)
          {
              Error = (Target - Output) * ActivationFunction.FunctionDeriv(Output);
          }
          else
          {
              Error = 0d;
              for(int i=0; i&lt;successor.NumNeurons; i++)
              {
                  Neuron neuron = successor.Neurons[i];
                  Error += (neuron.Error * m_weights[i] * ActivationFunction.FunctionDeriv(Output));
              }
          }
      }
  
      public void UpdateWeights(NeuronLayer successor)
      {
          if (MomentumAlpha != 0)
          {
              for (int i = 0; i &lt; successor.NumNeurons; i++)
              {
                  var neuron = successor.Neurons[i];
  
                  if (m_deltaWeights.Count &lt;= i)
                  {
                      double deltaWeight = LearningRate * neuron.Error * Output;
  
                      m_weights[i] += deltaWeight;
                      m_deltaWeights.Add(deltaWeight);
                  }
                  else
                  {
                      double deltaWeight = /*(1 - MomentumAlpha)*/LearningRate * neuron.Error * Output + MomentumAlpha * m_deltaWeights[i];
  
                      m_weights[i] += deltaWeight;
                      m_deltaWeights[i] = deltaWeight;
                  }
              }
          }
          else
          {
              for (int i = 0; i &lt; successor.NumNeurons; i++)
              {
                  var neuron = successor.Neurons[i];
                  double deltaWeight = LearningRate * neuron.Error * Output;
                  m_weights[i] += deltaWeight;
              }
          }
      }
  }
  </code></pre>
  
  <p>I hope some extra eyes and opinions helps find my problem. </p>
  
  <p>PS If you do download the source code from the repository and try run the application, please be aware that it will break if you don't set all necessary values before training, or if you press the reset button. I should get rid of the reset button, but I haven't yet. Sorry!</p>