# Set random seed:
srand(1234)




#uniform.jl

"""
Create a NxM uniformly distributed matrix using the method rand()*(b-a)-a where a,b are the Unif Parameters.

Arguments
---------
a
    Lower Bound 
b
    Upper Bound
N
    Row Dimension
M 
    Column Dimension

"""
function uniform(a,b,N,M)
    #Returns a NxM uniformly distributed matrix
    #for random number in range [a,b] we note that it can be generated by: rand() * (b-a) - a 
    #where rand() prints real numbers:
    rand(N,M)*(b-a)+a
end






#sigmoid.jl

"""
Return the sigmoid of x:

Arguments
---------
x
    input
h
    hyperparemeter

"""


function sigmoid(x,h)
    return 1/(1+exp(-h*x))
end


#sigmoid_deriv.jl

"""
Return the sigmoid of x:

Arguments
---------
x
    input
h
    hyperparemeter

"""


function sigmoid_deriv(x,h)
    return (1/(1+exp(-h*x)))*(1-1/(1+exp(-h*x)))
end




#Train_Neural_Net.jl

"""
Train the XOR neural net for a given number of epochs:

Arguments
---------
epochs
    number of times the network will foward and back propagate
Layer_1
    weighting for hidden layer 
Layer_2
    weighting for output layer 
learning_rate
    Learning Rate for Gradient Descent
"""

function Train_Neural_Net_Loop(epochs,Layer_1,Layer_2,learning_rate,node_function,node_deriv)

    #Initialise XOR truth values:
    X=[ 0 0 ; 0 1 ; 1 0 ; 1 1] #When selected X[i,:] we have column vectors so need to transpose for matmult
    Y=[ 0;1;1;0]

    MSE=0

    for i=1:epochs #For the weight matrix wij is the weight for input i going to Neuron j:

        s_1=map(node_function,X*Layer_1) #Applies Sigmoid to X*Layer_1 the weighting matrix
        s_2=map(node_function, s_1*Layer_2) #Applies Sigmoid to a2 * second weighting matrix
        direct_error=s_2-Y
        MSE=0.5*sum(direct_error.*direct_error) #element wise squares direct_error then summs up and divides by length


    #Begin updating: 
    delta_outer=-1.0*direct_error.*map(node_deriv,s_1*Layer_2) #this is the delta (which is the raw loss element-wise
                                                        #multiplied by z(3) see explanation in readme 
                                                        
    delta_inner=delta_outer*transpose(Layer_2).*map(node_deriv,X*Layer_1)  
    # map(sigmoid_deriv,X*Layer_1) is f'(z2)




    #Update the weights:

    Layer_2 +=learning_rate*transpose(s_1)*delta_outer #Outer layer has been updated. 
    Layer_1 += learning_rate*transpose(X)*delta_inner

  
    print(string(MSE,"\r"))

    
    end
    return Layer_1, Layer_2, MSE
    end








#Train_Neural_Net.jl

"""
Train the XOR neural net for a given mean square error it needs to reach:

Arguments
---------
MSE_Min
    Error the network trains till
Layer_1
    weighting for hidden layer 
Layer_2
    weighting for output layer 
learning_rate
    Learning Rate for Gradient Descent
"""

function Train_Neural_Net_MSE(MSE_Min,Layer_1,Layer_2,learning_rate,node_function,node_deriv)

    #Initialise XOR truth values:
    X=[ 0 0 ; 0 1 ; 1 0 ; 1 1] #When selected X[i,:] we have column vectors so need to transpose for matmult
    Y=[ 0;1;1;0]

    MSE=MSE_Min+1

    while MSE>=MSE_Min #For the weight matrix wij is the weight for input i going to Neuron j:

        s_1=map(node_function,X*Layer_1) #Applies Sigmoid to X*Layer_1 the weighting matrix
        s_2=map(node_function, s_1*Layer_2) #Applies Sigmoid to a2 * second weighting matrix
        direct_error=s_2-Y
        MSE=0.5*sum(direct_error.*direct_error) #element wise squares direct_error then summs up and divides by length


    #Begin updating: 
    delta_outer=-1.0*direct_error.*map(node_deriv,s_1*Layer_2) #this is the delta (which is the raw loss element-wise
                                                        #multiplied by z(3) see explanation in readme 
                                                        
    delta_inner=delta_outer*transpose(Layer_2).*map(node_deriv,X*Layer_1)  
    # map(sigmoid_deriv,X*Layer_1) is f'(z2)




    #Update the weights:

    Layer_2 +=learning_rate*transpose(s_1)*delta_outer #Outer layer has been updated. 
    Layer_1 += learning_rate*transpose(X)*delta_inner

  
    print(string(MSE,"\r"))

    
    end
    return Layer_1, Layer_2, MSE
    end







#XOR_Net.jl

"""
Output the values of a given XOR_Net

Arguments
---------
M
    Input for the XOR
w1
    Layer 1
w2
    Layer 2

"""

function XOR_Net(M,w1,w2,node_function)
    #Where M is a matrix that 
    s_1=map(node_function,M*w1) #Applies Sigmoid to X*Layer_1 the weighting matrix
    s_2=map(node_function,s_1*w2)
    return s_2

end



#-----------------------------Training Example--------------------------------#



# #Initialise Layers of Neuron Weights:


# Layer_1=uniform(0,1,2,2) #(In video above W1)
# Layer_2=uniform(0,1,2,1) #Column vector for the outer layer (in video above W2)






# function hyper_curry(h)
#     return (x->sigmoid(x,h))
# end

# function hyper_curry_deriv(h)
#     return (x->sigmoid_deriv(x,h))
# end

# node_function=hyper_curry(1)
# node_deriv=hyper_curry_deriv(1)

# w1, w2, MSE1 =Train_Neural_Net_Loop(1000000,Layer_1,Layer_2,0.01,node_function,node_deriv)

# print(XOR_Net([0 1],w1,w2,node_function))