import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import math
import os

# In this lab, we'll implement the basic functions of the Gradient Descent algorithm to find the boundary
# in a small dataset. 
# First, we'll start with some functions that will help us plot and visualize the data.
#

# Given two vectors, print them in a graph with matplotlib
def plot_points(X, y):
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'blue', edgecolor = 'k')
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'red', edgecolor = 'k')

# Plot the solution boundary
def display(m, b, color='g--'):
    plt.xlim(-0.05,1.05)
    plt.ylim(-0.05,1.05)
    x = np.arange(-10, 10, 0.1)
    plt.plot(x, m*x+b, color)

# Activation function (the sigmoid function, in our case)
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

# Output (prediction) formula. If we provide a neuron with weights, features and a bias
# this is the output it will be provided to the next layer(or as output if it is the last one)
def output_formula(features, weights, bias):
    return sigmoid(np.dot(features, weights) + bias)

# The formula that calculates the error if we use gradient descent for 'error fixing' 
def error_formula(y, output):
    return -y * np.log(output) - (1 - y) * np.log(1-output)

# Gradient descent step. This is how the weights are updated
def update_weights(x, y, weights, bias, learnrate):
    prediction = output_formula(x, weights, bias)
    diff_error = (y - prediction)
    weights += learnrate * diff_error * x
    bias += learnrate*diff_error
    return weights, bias

# The train function.
def train(features, targets, epochs, learnrate, graph_lines=False):
    errors = []
    n_records, n_features = features.shape
    last_loss = None
    # Start with random weights
    weights = np.random.normal(scale=1 / n_features**.5, size=n_features)
    bias = 0
    for e in range(epochs):
        for x, y in zip(features, targets):
            # The prediction for this node
            output = output_formula(x, weights, bias)
            # Given the expected value y, calculate the deviation from the actual value calculated
            error = error_formula(y, output)
            weights, bias = update_weights(x, y, weights, bias, learnrate)
        
        # We have updated the weights and the bias epoch times
        # Printing out the log-loss error on the training set
        # Now that we have the output of all neurons, calculate the end result from the NN
        out = output_formula(features, weights, bias)
        loss = np.mean(error_formula(targets, out))
        errors.append(loss)
        # Print the epochs' results in a 10 step rate
        if e % (epochs / 10) == 0:
            print("\n========== Epoch", e,"==========")
            if last_loss and last_loss < loss:
                print("Train loss: ", loss, "  WARNING - Loss Increasing")
            else:
                print("Train loss: ", loss)
            last_loss = loss
            predictions = out > 0.5
            accuracy = np.mean(predictions == targets)
            print("Accuracy: ", accuracy)
        if graph_lines and e % (epochs / 100) == 0:
            display(-weights[0]/weights[1], -bias/weights[1])
            

    # Plotting the solution boundary
    plt.title("Solution boundary")
    display(-weights[0]/weights[1], -bias/weights[1], 'black')

    # Plotting the data
    plot_points(features, targets)
    plt.show()

    # Plotting the error
    plt.title("Error Plot")
    plt.xlabel('Number of epochs')
    plt.ylabel('Error')
    plt.plot(errors)
    plt.show()

current_dir = os.path.dirname(__file__) 
data = pd.read_csv('{}/data.csv'.format(current_dir), header=None)
X = np.array(data[[0,1]])
y = np.array(data[2])
plot_points(X,y)
plt.show()

np.random.seed(44)
epochs = 100
learnrate = 0.01
train(X, y, epochs, learnrate, True)
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import math
import os

# In this lab, we'll implement the basic functions of the Gradient Descent algorithm to find the boundary
# in a small dataset. 
# First, we'll start with some functions that will help us plot and visualize the data.
#

# Given two vectors, print them in a graph with matplotlib
def plot_points(X, y):
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'blue', edgecolor = 'k')
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'red', edgecolor = 'k')

# Plot the solution boundary
def display(m, b, color='g--'):
    plt.xlim(-0.05,1.05)
    plt.ylim(-0.05,1.05)
    x = np.arange(-10, 10, 0.1)
    plt.plot(x, m*x+b, color)

# Activation function (the sigmoid function, in our case)
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

# Output (prediction) formula. If we provide a neuron with weights, features and a bias
# this is the output it will be provided to the next layer(or as output if it is the last one)
def output_formula(features, weights, bias):
    return sigmoid(np.dot(features, weights) + bias)

# The formula that calculates the error if we use gradient descent for 'error fixing' 
def error_formula(y, output):
    return -y * np.log(output) - (1 - y) * np.log(1-output)

# Gradient descent step. This is how the weights are updated
def update_weights(x, y, weights, bias, learnrate):
    prediction = output_formula(x, weights, bias)
    diff_error = (y - prediction)
    weights += learnrate * diff_error * x
    bias += learnrate*diff_error
    return weights, bias

# The train function.
def train(features, targets, epochs, learnrate, graph_lines=False):
    errors = []
    n_records, n_features = features.shape
    last_loss = None
    # Start with random weights
    weights = np.random.normal(scale=1 / n_features**.5, size=n_features)
    bias = 0
    for e in range(epochs):
        for x, y in zip(features, targets):
            # The prediction for this node
            output = output_formula(x, weights, bias)
            # Given the expected value y, calculate the deviation from the actual value calculated
            error = error_formula(y, output)
            weights, bias = update_weights(x, y, weights, bias, learnrate)
        
        # We have updated the weights and the bias epoch times
        # Printing out the log-loss error on the training set
        # Now that we have the output of all neurons, calculate the end result from the NN
        out = output_formula(features, weights, bias)
        loss = np.mean(error_formula(targets, out))
        errors.append(loss)
        # Print the epochs' results in a 10 step rate
        if e % (epochs / 10) == 0:
            print("\n========== Epoch", e,"==========")
            if last_loss and last_loss < loss:
                print("Train loss: ", loss, "  WARNING - Loss Increasing")
            else:
                print("Train loss: ", loss)
            last_loss = loss
            predictions = out > 0.5
            accuracy = np.mean(predictions == targets)
            print("Accuracy: ", accuracy)
        if graph_lines and e % (epochs / 100) == 0:
            display(-weights[0]/weights[1], -bias/weights[1])
            

    # Plotting the solution boundary
    plt.title("Solution boundary")
    display(-weights[0]/weights[1], -bias/weights[1], 'black')

    # Plotting the data
    plot_points(features, targets)
    plt.show()

    # Plotting the error
    plt.title("Error Plot")
    plt.xlabel('Number of epochs')
    plt.ylabel('Error')
    plt.plot(errors)
    plt.show()

current_dir = os.path.dirname(__file__) 
data = pd.read_csv('{}/data.csv'.format(current_dir), header=None)
X = np.array(data[[0,1]])
y = np.array(data[2])
plot_points(X,y)
plt.show()

np.random.seed(44)
epochs = 100
learnrate = 0.01
train(X, y, epochs, learnrate, True)
# In this lab, we'll implement the basic functions of the Gradient Descent algorithm to find the boundary # in a small dataset. # First, we'll start with some functions that will help us plot and visualize the data. # # Given two vectors, print them in a graph with matplotlib # Plot the solution boundary # Activation function (the sigmoid function, in our case) # Output (prediction) formula. If we provide a neuron with weights, features and a bias # this is the output it will be provided to the next layer(or as output if it is the last one) # The formula that calculates the error if we use gradient descent for 'error fixing' # Gradient descent step. This is how the weights are updated # The train function. # Start with random weights # The prediction for this node # Given the expected value y, calculate the deviation from the actual value calculated # We have updated the weights and the bias epoch times # Printing out the log-loss error on the training set # Now that we have the output of all neurons, calculate the end result from the NN # Print the epochs' results in a 10 step rate # Plotting the solution boundary # Plotting the data # Plotting the error