<jupyter_start><jupyter_text><jupyter_code>import scipy.io
import numpy as np
import matplotlib.pyplot as plt
#------------Activations-------------------------------------------------------
def sigmoid_kroko(Z):
    A = 1/(1+np.exp(-Z))
    cache = Z
    #np.exp statt math.exp da dies auch mit Vektoren geht
    #+Numbers between 0 and 1
    #-Prone to zero gradients
    #eher geeignet für letzten Layer
    return A, cache
def relu_kroko(Z):
    A = np.maximum(0,Z)
    cache = Z 
    return A, cache
def relu_backward_kroko(dA, cache):
    Z = cache
    dZ = np.array(dA, copy=True) # just converting dz to a correct object.
    dZ[Z <= 0] = 0
    return dZ
def sigmoid_backward_kroko(dA, cache):
    Z = cache
    s = 1/(1+np.exp(-Z))
    dZ = dA * s * (1-s)
    return dZ
#------------------------------------------------------------------------------
def initialize_parameters(layer_dims):#initialisierung a la He--> teilungsfaktor macht geringes W und damit größeren Gradienten
    np.random.seed(3)
    parameters = {}
    L = len(layer_dims) # number of layers in the network
    for l in range(1, L):
        parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1]) / np.sqrt(layer_dims[l-1])
        parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))
        assert(parameters['W' + str(l)].shape == (layer_dims[l], layer_dims[l-1]))
        assert(parameters['b' + str(l)].shape == (layer_dims[l], 1))
    return parameters
#------------------------------------------------------------------------------
def model(X, Y, layers_dims, learning_rate = 0.3, num_iterations = 30000, print_cost = True, lambd = 0, keep_prob = 1):  
    grads = {}
    costs = []                            # to keep track of the cost
    parameters = initialize_parameters(layers_dims)
    for i in range(0, num_iterations):
        AL, caches, Dropouts = forward_propagation(X, parameters, keep_prob)
        cost = compute_cost(AL, Y, caches, lambd)
        gradients = backward_propagation(AL, X, Y, caches, keep_prob, Dropouts, lambd)
        parameters = update_parameters(parameters, gradients, learning_rate)
        if print_cost and i % 10000 == 0: print("Cost after iteration {}: {}".format(i, cost))
        if print_cost and i % 1000 == 0: costs.append(cost)
    plt.figure("""first figure""")
    plt.plot(costs); plt.ylabel('cost');plt.xlabel('iterations (x1,000)');plt.title("Learning rate =" + str(learning_rate));plt.show()
    return parameters 
#------------------------------------------------------------------------------
def linear_forward(A, W, b):#A.shape=(n_l-1,m), d.h. X.shape=(n_1,m), W.shape=(n_l,n_l-1),b.shape=(n_l,1)
    Z = np.dot(W,A)+b#(n_l,n_l-1) * A.shape=(n_l-1,m) = (n_l,m)
    cache = (A, W, b)
    return Z, cache
def linear_activation_forward(A_prev, W, b, activation): #A.shape=(n_l,m), W.shape=(n_l,n_l-1),b.shape=(n_l,1)
    if activation == "sigmoid":
        Z, linear_cache = linear_forward(A_prev, W, b)
        A, activation_cache = sigmoid_kroko(Z)
    elif activation == "relu":
        Z, linear_cache = linear_forward(A_prev, W, b)
        A, activation_cache = relu_kroko(Z)
    cache = (linear_cache, activation_cache)
    return A, cache
def forward_propagation(X, parameters, keep_prob):
    caches = []
    Dropouts= []
    A = X
    L = len(parameters) // 2
    np.random.seed(1)
    for l in range(1, L):#1 bis L-1
        A_prev = A 
        A, cache = linear_activation_forward(A_prev, parameters['W' + str(l)], parameters['b' + str(l)], activation = "relu")
        D = np.random.rand(A.shape[0], A.shape[1])   #Dropout                                  
        D = D < keep_prob                            #Dropout                                               
        A = A * D                                    #Dropout     
        A = A / keep_prob                            #Dropout
        Dropouts.append(D)
        caches.append(cache)        #linear_cache, activation_cache = cache
    AL, cache = linear_activation_forward(A, parameters['W' + str(L)], parameters['b' + str(L)], activation = "sigmoid")
    caches.append(cache)
    return AL, caches, Dropouts
def linear_backward(dZ, cache, lambd):#dZ=(n_l,m)
    A_prev, W, b = cache #A_prev.shape=(n_l-1,m), W.shape=(n_l,n_l-1) b.shape=(n_l,1)
    m = A_prev.shape[1]
    dW = np.dot(dZ,A_prev.T)/m + lambd/m * W #dZ=(n_l,m) * (m*n_l-1) = (n_l,n_l-1) + (n_l,n_l-1)
    db = np.sum(dZ, axis=1,keepdims=True)/m #b.shape=(n_l,1)
    #keepdims=true sonst wird aus Spaltenvektor in Zeilenvektor, bzw die eine dimension fällt sonst raus
    dA_prev = np.dot(W.T,dZ)#(n_l-1,n_l) * (n_l,m) = (n_l-1 , m)
    return dA_prev, dW, db
def linear_activation_backward(dA, cache, activation, lambd):
    linear_cache, activation_cache = cache
    if activation == "relu":
        dZ = relu_backward_kroko(dA, activation_cache)
        dA_prev, dW, db = linear_backward(dZ, linear_cache, lambd)
    elif activation == "sigmoid":
        dZ = sigmoid_backward_kroko(dA, activation_cache)
        dA_prev, dW, db = linear_backward(dZ, linear_cache, lambd)
    return dA_prev, dW, db
#------------------------------------------------------------------------------
def backward_propagation(AL, X, Y, caches, keep_prob, Dropouts, lambd):
    L = len(caches)
    gradients = {}
    gradients["dZ" + str(L)] = AL - Y   #Die alternative dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL)) würde dich evtl durch null teilen lassen
    linear_cache, activation = caches[L-1]
    gradients["dA" + str(L-1)], gradients["dW" + str(L)], gradients["db" + str(L)]=linear_backward(gradients["dZ" + str(L)], linear_cache, lambd)
    gradients["dA" + str(L-1)] = gradients["dA" + str(L-1)] * Dropouts[L-2]/keep_prob
    for l in reversed(range(L-1)):
        current_cache = caches[l]
        gradients["dA" + str(l)], gradients["dW" + str(l+1)], gradients["db" + str(l+1)] = linear_activation_backward(gradients["dA" + str(l+1)], current_cache, "relu", lambd)
        if l>0: gradients["dA" + str(l)]= gradients["dA" + str(l)] * Dropouts[l-1]/ keep_prob #dA0 bekommt kein Dropout 
    return gradients
#------------------------------------------------------------------------------
def compute_cost(AL, Y, caches, lambd):#A.shape=(n_L,m), Y.shape=(n_L,m)
    m = Y.shape[1]
    L=len(caches)
    cross_entropy = np.nansum(-(Y*np.log(AL)+(1-Y)*np.log(1-AL)),axis=1)/m#Kostenfuntion für Klassifizierung zw. 0 und 1
    L2_regularization=0
    for l in range(0,L):
        (linear_cache, activation_cache)=caches[l]
        A,W,b = linear_cache
        L2_regularization += np.nansum(np.square(W)) * 1/m * lambd/2
    cost = cross_entropy+L2_regularization
    cost = np.squeeze(cost)#Dimensionen mit nur einem Eintrag werden gelöscht, i.e. aus [[17]] wird 17
    return cost
def update_parameters(parameters, grads, learning_rate):
    n = len(parameters) // 2 # number of layers in the neural networks
    for k in range(n):
        parameters["W" + str(k+1)] = parameters["W" + str(k+1)] - learning_rate * grads["dW" + str(k+1)]
        parameters["b" + str(k+1)] = parameters["b" + str(k+1)] - learning_rate * grads["db" + str(k+1)]
    return parameters
def predict(X, y, parameters):
    m = X.shape[1]
    p = np.zeros((1,m), dtype = np.int)
    AL, caches, _ = forward_propagation(X, parameters, keep_prob=1.0)
    for i in range(0, AL.shape[1]):
        if AL[0,i] > 0.5: p[0,i] = 1
        else: p[0,i] = 0
    print("Accuracy: "  + str(np.mean((p[0,:] == y[0,:]))))
    return p
#------------------------------------------------------------------------------
data = scipy.io.loadmat('/content/drive/MyDrive/Colab Notebooks/PureNumpy/NeuronalNetwork-binaryClassification/data.mat')
train_X = data['X'].T
train_Y = data['y'].T
test_X = data['Xval'].T
test_Y = data['yval'].T

plt.figure("""second figure""")
plt.scatter(train_X[0, :], train_X[1, :], c=train_Y[0,:], s=40, cmap=plt.cm.Spectral);

layers_dims = [train_X.shape[0], 40, 5, 3, 1]
print("train_X.shape: "+str(train_X.shape))
print("train_Y.shape: "+str(train_Y.shape))
parameters = model(train_X, train_Y, layers_dims,  keep_prob = 1, learning_rate = 0.09, lambd=0.7)
print ("On the train set:")
predictions_train = predict(train_X, train_Y, parameters)
print ("On the test set:")
predictions_test = predict(test_X, test_Y, parameters)<jupyter_output>train_X.shape: (2, 211)
train_Y.shape: (1, 211)
Cost after iteration 0: 0.7760264977289945
Cost after iteration 10000: 0.24669274317839368
Cost after iteration 20000: 0.24372267094983083