<reponame>willcanniford/python-notes
import pandas as pd
import numpy as np


def sigmoid(x):
    """
    Calculate sigmoid
    """
    return 1 / (1 + np.exp(-x))


def sigmoid_prime(x):
    """
    # Derivative of the sigmoid function
    """
    return sigmoid(x) * (1 - sigmoid(x))


# Define the learning rate and inputs/result for our example
learnrate = 0.5
x = np.array([1, 2, 3, 4])
y = np.array(0.5)

# Initial weights
w = np.array([0.5, -0.5, 0.3, 0.1])

# Calculate one gradient descent step for each weight

# TODO: Calculate the node's linear combination of inputs and weights
h = np.dot(x, w)

# TODO: Calculate output of neural network
# This would be the sigmoid probability of a positive result given the
# current weights
nn_output = sigmoid(h)

# TODO: Calculate error of neural network
# How far away was our initial prediction?
error = y - nn_output

# TODO: Calculate the error term
#       Remember, this requires the output gradient, which we haven't
#       specifically added a variable for.
error_term = error * sigmoid_prime(h)

# TODO: Calculate change in weights
# Learn rate controls the size of the steps we take
# Error term tells us in which direction to alter the weights
del_w = learnrate * error_term * x

print('Neural Network output:')
print(nn_output)
print('Amount of Error:')
print(error)
print('Change in Weights:')
print(del_w)


# Another example of implement gradient descent from exercise
# Data prep examples

admissions = pd.read_csv('binary.csv')

# Make dummy variables for rank
data = pd.concat([admissions, pd.get_dummies(
    admissions['rank'], prefix='rank')], axis=1)
data = data.drop('rank', axis=1)

# Standarize features
for field in ['gre', 'gpa']:
    mean, std = data[field].mean(), data[field].std()
    data.loc[:, field] = (data[field] - mean) / std

# Split off random 10% of the data for testing
np.random.seed(42)
sample = np.random.choice(data.index, size=int(len(data) * 0.9), replace=False)
data, test_data = data.ix[sample], data.drop(sample)

# Split into features and targets
features, targets = data.drop('admit', axis=1), data['admit']
features_test, targets_test = test_data.drop(
    'admit', axis=1), test_data['admit']


# Implement the gradient descent steps


def sigmoid(x):
    """
    Calculate sigmoid function
    """
    return 1 / (1 + np.exp(-x))


# Use to same seed to make debugging easier
np.random.seed(42)

n_records, n_features = features.shape
last_loss = None

# Initialize weights
weights = np.random.normal(scale=1 / n_features**.5, size=n_features)

# Neural Network hyperparameters
epochs = 1000
learnrate = 0.5

for e in range(epochs):
    del_w = np.zeros(weights.shape)
    for x, y in zip(features.values, targets):
        # Loop through all records, x is the input, y is the target

        # Note: We haven't included the h variable from the previous
        #       lesson. You can add it if you want, or you can calculate
        #       the h together with the output

        # Calculate the output from our h value (weights and inputs)
        output = sigmoid(np.dot(weights, x))

        # Calculate the error
        error = y - output

        # Calculate the error term
        error_term = error * output * (1 - output)

        # Calculate the change in weights for this sample and add it to the total weight change
        # Note here that the del stands for delta, i.e. the change in the
        # weights
        del_w += error * x

    # TODO: Update weights using the learning rate and the average change in weights
    # This is because we are calculating it point by point as opposed to
    # calculating the average error across all points
    weights += learnrate * del_w / n_records

    # Printing out the mean square error on the training set
    if e % (epochs / 10) == 0:
        out = sigmoid(np.dot(features, weights))
        loss = np.mean((out - targets) ** 2)
        if last_loss and last_loss < loss:
            print("Train loss: ", loss, "  WARNING - Loss Increasing")
        else:
            print("Train loss: ", loss)
        last_loss = loss


# Calculate accuracy on test data
tes_out = sigmoid(np.dot(features_test, weights))
predictions = tes_out > 0.5
accuracy = np.mean(predictions == targets_test)
print("Prediction accuracy: {:.3f}".format(accuracy))

import pandas as pd
import numpy as np


def sigmoid(x):
    """
    Calculate sigmoid
    """
    return 1 / (1 + np.exp(-x))


def sigmoid_prime(x):
    """
    # Derivative of the sigmoid function
    """
    return sigmoid(x) * (1 - sigmoid(x))


# Define the learning rate and inputs/result for our example
learnrate = 0.5
x = np.array([1, 2, 3, 4])
y = np.array(0.5)

# Initial weights
w = np.array([0.5, -0.5, 0.3, 0.1])

# Calculate one gradient descent step for each weight

# TODO: Calculate the node's linear combination of inputs and weights
h = np.dot(x, w)

# TODO: Calculate output of neural network
# This would be the sigmoid probability of a positive result given the
# current weights
nn_output = sigmoid(h)

# TODO: Calculate error of neural network
# How far away was our initial prediction?
error = y - nn_output

# TODO: Calculate the error term
#       Remember, this requires the output gradient, which we haven't
#       specifically added a variable for.
error_term = error * sigmoid_prime(h)

# TODO: Calculate change in weights
# Learn rate controls the size of the steps we take
# Error term tells us in which direction to alter the weights
del_w = learnrate * error_term * x

print('Neural Network output:')
print(nn_output)
print('Amount of Error:')
print(error)
print('Change in Weights:')
print(del_w)


# Another example of implement gradient descent from exercise
# Data prep examples

admissions = pd.read_csv('binary.csv')

# Make dummy variables for rank
data = pd.concat([admissions, pd.get_dummies(
    admissions['rank'], prefix='rank')], axis=1)
data = data.drop('rank', axis=1)

# Standarize features
for field in ['gre', 'gpa']:
    mean, std = data[field].mean(), data[field].std()
    data.loc[:, field] = (data[field] - mean) / std

# Split off random 10% of the data for testing
np.random.seed(42)
sample = np.random.choice(data.index, size=int(len(data) * 0.9), replace=False)
data, test_data = data.ix[sample], data.drop(sample)

# Split into features and targets
features, targets = data.drop('admit', axis=1), data['admit']
features_test, targets_test = test_data.drop(
    'admit', axis=1), test_data['admit']


# Implement the gradient descent steps


def sigmoid(x):
    """
    Calculate sigmoid function
    """
    return 1 / (1 + np.exp(-x))


# Use to same seed to make debugging easier
np.random.seed(42)

n_records, n_features = features.shape
last_loss = None

# Initialize weights
weights = np.random.normal(scale=1 / n_features**.5, size=n_features)

# Neural Network hyperparameters
epochs = 1000
learnrate = 0.5

for e in range(epochs):
    del_w = np.zeros(weights.shape)
    for x, y in zip(features.values, targets):
        # Loop through all records, x is the input, y is the target

        # Note: We haven't included the h variable from the previous
        #       lesson. You can add it if you want, or you can calculate
        #       the h together with the output

        # Calculate the output from our h value (weights and inputs)
        output = sigmoid(np.dot(weights, x))

        # Calculate the error
        error = y - output

        # Calculate the error term
        error_term = error * output * (1 - output)

        # Calculate the change in weights for this sample and add it to the total weight change
        # Note here that the del stands for delta, i.e. the change in the
        # weights
        del_w += error * x

    # TODO: Update weights using the learning rate and the average change in weights
    # This is because we are calculating it point by point as opposed to
    # calculating the average error across all points
    weights += learnrate * del_w / n_records

    # Printing out the mean square error on the training set
    if e % (epochs / 10) == 0:
        out = sigmoid(np.dot(features, weights))
        loss = np.mean((out - targets) ** 2)
        if last_loss and last_loss < loss:
            print("Train loss: ", loss, "  WARNING - Loss Increasing")
        else:
            print("Train loss: ", loss)
        last_loss = loss


# Calculate accuracy on test data
tes_out = sigmoid(np.dot(features_test, weights))
predictions = tes_out > 0.5
accuracy = np.mean(predictions == targets_test)
print("Prediction accuracy: {:.3f}".format(accuracy))
Calculate sigmoid # Derivative of the sigmoid function # Define the learning rate and inputs/result for our example # Initial weights # Calculate one gradient descent step for each weight # TODO: Calculate the node's linear combination of inputs and weights # TODO: Calculate output of neural network # This would be the sigmoid probability of a positive result given the # current weights # TODO: Calculate error of neural network # How far away was our initial prediction? # TODO: Calculate the error term #       Remember, this requires the output gradient, which we haven't #       specifically added a variable for. # TODO: Calculate change in weights # Learn rate controls the size of the steps we take # Error term tells us in which direction to alter the weights # Another example of implement gradient descent from exercise # Data prep examples # Make dummy variables for rank # Standarize features # Split off random 10% of the data for testing # Split into features and targets # Implement the gradient descent steps Calculate sigmoid function # Use to same seed to make debugging easier # Initialize weights # Neural Network hyperparameters # Loop through all records, x is the input, y is the target # Note: We haven't included the h variable from the previous #       lesson. You can add it if you want, or you can calculate #       the h together with the output # Calculate the output from our h value (weights and inputs) # Calculate the error # Calculate the error term # Calculate the change in weights for this sample and add it to the total weight change # Note here that the del stands for delta, i.e. the change in the # weights # TODO: Update weights using the learning rate and the average change in weights # This is because we are calculating it point by point as opposed to # calculating the average error across all points # Printing out the mean square error on the training set # Calculate accuracy on test data