{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class NeuralNetwork(object):\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):\n",
    "        # Set number of nodes in input, hidden and output layers.\n",
    "        self.input_nodes = input_nodes\n",
    "        self.hidden_nodes = hidden_nodes\n",
    "        self.output_nodes = output_nodes\n",
    "\n",
    "        # Initialize weights\n",
    "        self.weights_input_to_hidden = np.random.normal(0.0, self.input_nodes**-0.5, \n",
    "                                       (self.input_nodes, self.hidden_nodes))\n",
    "\n",
    "        self.weights_hidden_to_output = np.random.normal(0.0, self.hidden_nodes**-0.5, \n",
    "                                       (self.hidden_nodes, self.output_nodes))\n",
    "        self.lr = learning_rate\n",
    "        \n",
    "        #### TODO: Set self.activation_function to your implemented sigmoid function ####\n",
    "        #\n",
    "        # Note: in Python, you can define a function with a lambda expression,\n",
    "        # as shown below.\n",
    "        self.activation_function = lambda x : 1/(1+np.exp(-x))  # Replace 0 with your sigmoid calculation.\n",
    "        \n",
    "        ### If the lambda code above is not something you're familiar with,\n",
    "        # You can uncomment out the following three lines and put your \n",
    "        # implementation there instead.\n",
    "        #\n",
    "        #def sigmoid(x):\n",
    "        #    return 0  # Replace 0 with your sigmoid calculation here\n",
    "        #self.activation_function = sigmoid\n",
    "                    \n",
    "\n",
    "    def train(self, features, targets):\n",
    "        ''' Train the network on batch of features and targets. \n",
    "        \n",
    "            Arguments\n",
    "            ---------\n",
    "            \n",
    "            features: 2D array, each row is one data record, each column is a feature\n",
    "            targets: 1D array of target values\n",
    "        \n",
    "        '''\n",
    "        n_records = features.shape[0]\n",
    "        delta_weights_i_h = np.zeros(self.weights_input_to_hidden.shape)\n",
    "        delta_weights_h_o = np.zeros(self.weights_hidden_to_output.shape)\n",
    "        for X, y in zip(features, targets):\n",
    "            \n",
    "            final_outputs, hidden_outputs = self.forward_pass_train(X)  # Implement the forward pass function below\n",
    "            # Implement the backproagation function below\n",
    "            delta_weights_i_h, delta_weights_h_o = self.backpropagation(final_outputs, hidden_outputs, X, y, \n",
    "                                                                        delta_weights_i_h, delta_weights_h_o)\n",
    "        self.update_weights(delta_weights_i_h, delta_weights_h_o, n_records)\n",
    "\n",
    "\n",
    "    def forward_pass_train(self, X):\n",
    "        ''' Implement forward pass here \n",
    "         \n",
    "            Arguments\n",
    "            ---------\n",
    "            X: features batch\n",
    "\n",
    "        '''\n",
    "        #### Implement the forward pass here ####\n",
    "        ### Forward pass ###\n",
    "        # TODO: Hidden layer - Replace these values with your calculations.\n",
    "        hidden_inputs = np.dot(X, self.weights_input_to_hidden) # signals into hidden layer\n",
    "        hidden_outputs = self.activation_function(hidden_inputs) # signals from hidden layer\n",
    "\n",
    "        # TODO: Output layer - Replace these values with your calculations.\n",
    "        final_inputs = np.dot(hidden_outputs, self.weights_hidden_to_output) # signals into final output layer\n",
    "        final_outputs = final_inputs # signals from final output layer\n",
    "        \n",
    "        return final_outputs, hidden_outputs\n",
    "\n",
    "    def backpropagation(self, final_outputs, hidden_outputs, X, y, delta_weights_i_h, delta_weights_h_o):\n",
    "        ''' Implement backpropagation\n",
    "         \n",
    "            Arguments\n",
    "            ---------\n",
    "            final_outputs: output from forward pass\n",
    "            y: target (i.e. label) batch\n",
    "            delta_weights_i_h: change in weights from input to hidden layers\n",
    "            delta_weights_h_o: change in weights from hidden to output layers\n",
    "\n",
    "        '''\n",
    "        #### Implement the backward pass here ####\n",
    "        ### Backward pass ###\n",
    "\n",
    "        # TODO: Output error - Replace this value with your calculations.\n",
    "        error = y - final_outputs # Output layer error is the difference between desired target and actual output.\n",
    "        \n",
    "        # TODO: Calculate the hidden layer's contribution to the error\n",
    "        hidden_error = np.dot(error, self.weights_hidden_to_output.T)\n",
    "        \n",
    "        # TODO: Backpropagated error terms - Replace these values with your calculations.\n",
    "        output_error_term = error\n",
    "        \n",
    "        hidden_error_term = hidden_error * hidden_outputs * (1 - hidden_outputs)\n",
    "        \n",
    "        # Weight step (input to hidden)\n",
    "        delta_weights_i_h += hidden_error_term * X[:, None]\n",
    "        # Weight step (hidden to output)\n",
    "        delta_weights_h_o += output_error_term * hidden_outputs[:, None]\n",
    "        return delta_weights_i_h, delta_weights_h_o\n",
    "\n",
    "    def update_weights(self, delta_weights_i_h, delta_weights_h_o, n_records):\n",
    "        ''' Update weights on gradient descent step\n",
    "         \n",
    "            Arguments\n",
    "            ---------\n",
    "            delta_weights_i_h: change in weights from input to hidden layers\n",
    "            delta_weights_h_o: change in weights from hidden to output layers\n",
    "            n_records: number of records\n",
    "\n",
    "        '''\n",
    "        self.weights_hidden_to_output += self.lr * delta_weights_h_o / n_records # update hidden-to-output weights with gradient descent step\n",
    "        self.weights_input_to_hidden += self.lr * delta_weights_i_h / n_records # update input-to-hidden weights with gradient descent step\n",
    "\n",
    "    def run(self, features):\n",
    "        ''' Run a forward pass through the network with input features \n",
    "        \n",
    "            Arguments\n",
    "            ---------\n",
    "            features: 1D array of feature values\n",
    "        '''\n",
    "        \n",
    "        #### Implement the forward pass here ####\n",
    "        # TODO: Hidden layer - replace these values with the appropriate calculations.\n",
    "        hidden_inputs = np.dot(features, self.weights_input_to_hidden) # signals into hidden layer\n",
    "        hidden_outputs = self.activation_function(hidden_inputs) # signals from hidden layer\n",
    "        \n",
    "        # TODO: Output layer - Replace these values with the appropriate calculations.\n",
    "        final_inputs = np.dot(hidden_outputs, self.weights_hidden_to_output) # signals into final output layer\n",
    "        final_outputs = final_inputs # signals from final output layer \n",
    "        \n",
    "        return final_outputs\n",
    "\n",
    "\n",
    "#########################################################\n",
    "# Set your hyperparameters here\n",
    "##########################################################\n",
    "iterations = 100\n",
    "learning_rate = 0.1\n",
    "hidden_nodes = 2\n",
    "output_nodes = 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "class NeuralNetwork(object):\n",
    "    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):\n",
    "        # Set number of nodes in input, hidden and output layers.\n",
    "        self.input_nodes = input_nodes\n",
    "        self.hidden_nodes = hidden_nodes\n",
    "        self.output_nodes = output_nodes\n",
    "\n",
    "        # Initialize weights\n",
    "        self.weights_input_to_hidden = np.random.normal(0.0, self.input_nodes**-0.5, \n",
    "                                       (self.input_nodes, self.hidden_nodes))\n",
    "\n",
    "        self.weights_hidden_to_output = np.random.normal(0.0, self.hidden_nodes**-0.5, \n",
    "                                       (self.hidden_nodes, self.output_nodes))\n",
    "        self.lr = learning_rate\n",
    "        \n",
    "        #### TODO: Set self.activation_function to your implemented sigmoid function ####\n",
    "        #\n",
    "        # Note: in Python, you can define a function with a lambda expression,\n",
    "        # as shown below.\n",
    "        self.activation_function = lambda x : 1/(1+np.exp(-x))  # Replace 0 with your sigmoid calculation.\n",
    "        \n",
    "        ### If the lambda code above is not something you're familiar with,\n",
    "        # You can uncomment out the following three lines and put your \n",
    "        # implementation there instead.\n",
    "        #\n",
    "        #def sigmoid(x):\n",
    "        #    return 0  # Replace 0 with your sigmoid calculation here\n",
    "        #self.activation_function = sigmoid\n",
    "                    \n",
    "\n",
    "    def train(self, features, targets):\n",
    "        ''' Train the network on batch of features and targets. \n",
    "        \n",
    "            Arguments\n",
    "            ---------\n",
    "            \n",
    "            features: 2D array, each row is one data record, each column is a feature\n",
    "            targets: 1D array of target values\n",
    "        \n",
    "        '''\n",
    "        n_records = features.shape[0]\n",
    "        delta_weights_i_h = np.zeros(self.weights_input_to_hidden.shape)\n",
    "        delta_weights_h_o = np.zeros(self.weights_hidden_to_output.shape)\n",
    "        for X, y in zip(features, targets):\n",
    "            \n",
    "            final_outputs, hidden_outputs = self.forward_pass_train(X)  # Implement the forward pass function below\n",
    "            # Implement the backproagation function below\n",
    "            delta_weights_i_h, delta_weights_h_o = self.backpropagation(final_outputs, hidden_outputs, X, y, \n",
    "                                                                        delta_weights_i_h, delta_weights_h_o)\n",
    "        self.update_weights(delta_weights_i_h, delta_weights_h_o, n_records)\n",
    "\n",
    "\n",
    "    def forward_pass_train(self, X):\n",
    "        ''' Implement forward pass here \n",
    "         \n",
    "            Arguments\n",
    "            ---------\n",
    "            X: features batch\n",
    "\n",
    "        '''\n",
    "        #### Implement the forward pass here ####\n",
    "        ### Forward pass ###\n",
    "        # TODO: Hidden layer - Replace these values with your calculations.\n",
    "        hidden_inputs = np.dot(X, self.weights_input_to_hidden) # signals into hidden layer\n",
    "        hidden_outputs = self.activation_function(hidden_inputs) # signals from hidden layer\n",
    "\n",
    "        # TODO: Output layer - Replace these values with your calculations.\n",
    "        final_inputs = np.dot(hidden_outputs, self.weights_hidden_to_output) # signals into final output layer\n",
    "        final_outputs = final_inputs # signals from final output layer\n",
    "        \n",
    "        return final_outputs, hidden_outputs\n",
    "\n",
    "    def backpropagation(self, final_outputs, hidden_outputs, X, y, delta_weights_i_h, delta_weights_h_o):\n",
    "        ''' Implement backpropagation\n",
    "         \n",
    "            Arguments\n",
    "            ---------\n",
    "            final_outputs: output from forward pass\n",
    "            y: target (i.e. label) batch\n",
    "            delta_weights_i_h: change in weights from input to hidden layers\n",
    "            delta_weights_h_o: change in weights from hidden to output layers\n",
    "\n",
    "        '''\n",
    "        #### Implement the backward pass here ####\n",
    "        ### Backward pass ###\n",
    "\n",
    "        # TODO: Output error - Replace this value with your calculations.\n",
    "        error = y - final_outputs # Output layer error is the difference between desired target and actual output.\n",
    "        \n",
    "        # TODO: Calculate the hidden layer's contribution to the error\n",
    "        hidden_error = np.dot(error, self.weights_hidden_to_output.T)\n",
    "        \n",
    "        # TODO: Backpropagated error terms - Replace these values with your calculations.\n",
    "        output_error_term = error\n",
    "        \n",
    "        hidden_error_term = hidden_error * hidden_outputs * (1 - hidden_outputs)\n",
    "        \n",
    "        # Weight step (input to hidden)\n",
    "        delta_weights_i_h += hidden_error_term * X[:, None]\n",
    "        # Weight step (hidden to output)\n",
    "        delta_weights_h_o += output_error_term * hidden_outputs[:, None]\n",
    "        return delta_weights_i_h, delta_weights_h_o\n",
    "\n",
    "    def update_weights(self, delta_weights_i_h, delta_weights_h_o, n_records):\n",
    "        ''' Update weights on gradient descent step\n",
    "         \n",
    "            Arguments\n",
    "            ---------\n",
    "            delta_weights_i_h: change in weights from input to hidden layers\n",
    "            delta_weights_h_o: change in weights from hidden to output layers\n",
    "            n_records: number of records\n",
    "\n",
    "        '''\n",
    "        self.weights_hidden_to_output += self.lr * delta_weights_h_o / n_records # update hidden-to-output weights with gradient descent step\n",
    "        self.weights_input_to_hidden += self.lr * delta_weights_i_h / n_records # update input-to-hidden weights with gradient descent step\n",
    "\n",
    "    def run(self, features):\n",
    "        ''' Run a forward pass through the network with input features \n",
    "        \n",
    "            Arguments\n",
    "            ---------\n",
    "            features: 1D array of feature values\n",
    "        '''\n",
    "        \n",
    "        #### Implement the forward pass here ####\n",
    "        # TODO: Hidden layer - replace these values with the appropriate calculations.\n",
    "        hidden_inputs = np.dot(features, self.weights_input_to_hidden) # signals into hidden layer\n",
    "        hidden_outputs = self.activation_function(hidden_inputs) # signals from hidden layer\n",
    "        \n",
    "        # TODO: Output layer - Replace these values with the appropriate calculations.\n",
    "        final_inputs = np.dot(hidden_outputs, self.weights_hidden_to_output) # signals into final output layer\n",
    "        final_outputs = final_inputs # signals from final output layer \n",
    "        \n",
    "        return final_outputs\n",
    "\n",
    "\n",
    "#########################################################\n",
    "# Set your hyperparameters here\n",
    "##########################################################\n",
    "iterations = 100\n",
    "learning_rate = 0.1\n",
    "hidden_nodes = 2\n",
    "output_nodes = 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

# Set number of nodes in input, hidden and output layers.\n", # Initialize weights\n", #### TODO: Set self.activation_function to your implemented sigmoid function ####\n", #\n", # Note: in Python, you can define a function with a lambda expression,\n", # as shown below.\n", # Replace 0 with your sigmoid calculation.\n", ### If the lambda code above is not something you're familiar with,\n", # You can uncomment out the following three lines and put your \n", # implementation there instead.\n", #\n", #def sigmoid(x):\n", #    return 0  # Replace 0 with your sigmoid calculation here\n", #self.activation_function = sigmoid\n", Train the network on batch of features and targets. \n",     "        \n",     "            Arguments\n",     "            ---------\n",     "            \n",     "            features: 2D array, each row is one data record, each column is a feature\n",     "            targets: 1D array of target values\n",     "        \n",     " # Implement the forward pass function below\n", # Implement the backproagation function below\n", Implement forward pass here \n",     "         \n",     "            Arguments\n",     "            ---------\n",     "            X: features batch\n",     "\n",     " #### Implement the forward pass here ####\n", ### Forward pass ###\n", # TODO: Hidden layer - Replace these values with your calculations.\n", # signals into hidden layer\n", # signals from hidden layer\n", # TODO: Output layer - Replace these values with your calculations.\n", # signals into final output layer\n", # signals from final output layer\n", Implement backpropagation\n",     "         \n",     "            Arguments\n",     "            ---------\n",     "            final_outputs: output from forward pass\n",     "            y: target (i.e. label) batch\n",     "            delta_weights_i_h: change in weights from input to hidden layers\n",     "            delta_weights_h_o: change in weights from hidden to output layers\n",     "\n",     " #### Implement the backward pass here ####\n", ### Backward pass ###\n", # TODO: Output error - Replace this value with your calculations.\n", # Output layer error is the difference between desired target and actual output.\n", # TODO: Calculate the hidden layer's contribution to the error\n", # TODO: Backpropagated error terms - Replace these values with your calculations.\n", # Weight step (input to hidden)\n", # Weight step (hidden to output)\n", Update weights on gradient descent step\n",     "         \n",     "            Arguments\n",     "            ---------\n",     "            delta_weights_i_h: change in weights from input to hidden layers\n",     "            delta_weights_h_o: change in weights from hidden to output layers\n",     "            n_records: number of records\n",     "\n",     " # update hidden-to-output weights with gradient descent step\n", # update input-to-hidden weights with gradient descent step\n", Run a forward pass through the network with input features \n",     "        \n",     "            Arguments\n",     "            ---------\n",     "            features: 1D array of feature values\n",     " #### Implement the forward pass here ####\n", # TODO: Hidden layer - replace these values with the appropriate calculations.\n", # signals into hidden layer\n", # signals from hidden layer\n", # TODO: Output layer - Replace these values with the appropriate calculations.\n", # signals into final output layer\n", # signals from final output layer \n", ########################################################\n", #########################################################\n",