```python
import numpy as np
from keras import optimizers

X = np.array([[0,0],[0,1],[1,0],[1,1]])
y = np.array([0,1,1,0])

from keras import models
from keras import layers

model = models.Sequential() 
model.add(layers.Dense(2, activation='tanh'))
model.add(layers.Dense(1, activation='sigmoid'))
sgd = optimizers.SGD(lr=0.1)
model.compile(optimizer=sgd, loss='binary_crossentropy')
model.fit(X, y, epochs=1000, batch_size=1, verbose=0)
model.predict(X)

```




    array([[0.0076664 ],
           [0.9866918 ],
           [0.98638123],
           [0.00661352]], dtype=float32)



\begin{equation}
    \frac{\partial L}{\partial \vec{w}^{\,2}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial {w}^{2}_{11}}\\
        \\
        \frac{\partial L}{\partial {w}^{2}_{21}} \\
    \end{bmatrix}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{2}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {w}^{2}_{11}}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {\hat{y}}_{1}} =  (\frac{1-y}{(1-\hat{y}_{1})} - \frac{y}{\hat{y}_{1}  })
\end{equation}

\begin{equation}
    \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}} = 1
\end{equation}

\begin{equation}
    \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}} = 
    \frac{\partial}{\partial {a}^{2}_{1}}
    \cdot \sigma ({a}^{2}_{1}) = \sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))
\end{equation}
    
\begin{equation}
    \frac{\partial {a}^{2}_{1}}{\partial {w}^{2}_{11}} = \frac{\partial}{\partial {w}^{2}_{11}} \cdot ({h}^{1}_{1} \cdot {w}^{2}_{11} + {h}^{1}_{2} \cdot {w}^{2}_{21} + b^{2}) = {h}^{1}_{1}
\end{equation}

\begin{equation}
   \frac{\partial L}{\partial {w}^{2}_{11}} = (\frac{1-y}{(1-\hat{y}_{1})} - \frac{y}{\hat{y}_{1}  }) \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
   \cdot {h}^{1}_{1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{2}_{21}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {w}^{2}_{21}} = 
    (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
   \cdot {h}^{1}_{2})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {b}^{2}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {b}^{2}} = 
    (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
   \cdot 1)
\end{equation}

LAYER 1
\begin{equation}
        \frac{\partial L}{\partial \vec{w}^{\,1}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial {w}^{1}_{11}} && \frac{\partial L}{\partial {w}^{1}_{12}}\\
        \\
        \frac{\partial L}{\partial {w}^{1}_{21}}  && \frac{\partial L}{\partial {w}^{1}_{22}}\\
    \end{bmatrix}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{1}}
    \cdot \frac{\partial {h}^{1}_{1}}{\partial {a}^{1}_{1}}
    \cdot \frac{\partial {a}^{1}_{1}}{\partial {w}^{1}_{11}} 
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1 
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{11}
    \cdot [1-\tanh{({a}^{1}_{1})}]
   \cdot {h}^{0}_{1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{2}}
    \cdot \frac{\partial {h}^{1}_{2}}{\partial {a}^{1}_{2}}
    \cdot \frac{\partial {a}^{1}_{2}}{\partial {w}^{1}_{12}} 
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1 
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{21}
    \cdot [1-\tanh{({a}^{1}_{2})}]
   \cdot {h}^{0}_{1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{1}}
    \cdot \frac{\partial {h}^{1}_{1}}{\partial {a}^{1}_{1}}
    \cdot \frac{\partial {a}^{1}_{1}}{\partial {w}^{1}_{21}} 
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1 
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{11}
    \cdot [1-\tanh{({a}^{1}_{1})}]
   \cdot {h}^{0}_{2})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {w}^{1}_{11}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{2}}
    \cdot \frac{\partial {h}^{1}_{2}}{\partial {a}^{1}_{2}}
    \cdot \frac{\partial {a}^{1}_{2}}{\partial {w}^{1}_{22}} 
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1 
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{21}
    \cdot [1-\tanh{({a}^{1}_{2})}]
   \cdot {h}^{0}_{2})
\end{equation}

BIASES

\begin{equation}
        \frac{\partial L}{\partial \vec{b}^{\,1}} =
    \begin{bmatrix}
        \frac{\partial L}{\partial {b}^{1}_{1}}\\
        \\
        \frac{\partial L}{\partial {b}^{1}_{2}}\\
    \end{bmatrix}
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {b}^{1}_{1}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{1}}
    \cdot \frac{\partial {h}^{1}_{1}}{\partial {a}^{1}_{1}}
    \cdot \frac{\partial {a}^{1}_{1}}{\partial {b}^{1}_{1}} 
    = (\frac{1-y}{(1-\hat{y}_{1})  } - \frac{y}{\hat{y}_{1}  }) \cdot 1 
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{11}
    \cdot [1-\tanh{({a}^{1}_{1})}]
   \cdot {1})
\end{equation}

\begin{equation}
    \frac{\partial L}{\partial {b}^{1}_{2}} = \frac{\partial L}{\partial {\hat{y}}_{1}} 
    \cdot \frac{\partial {\hat{y}}_{1}}{\partial {h}^{2}_{1}}
    \cdot \frac{\partial {h}^{2}_{1}}{\partial {a}^{2}_{1}}
    \cdot \frac{\partial {a}^{2}_{1}}{\partial {h}^{1}_{2}}
    \cdot \frac{\partial {h}^{1}_{2}}{\partial {a}^{1}_{2}}
    \cdot \frac{\partial {a}^{1}_{2}}{\partial {b}^{1}_{2}} 
    = (\frac{1-y}{(1-\hat{y}_{1})} - \frac{y}{\hat{y}_{1}}) \cdot 1 
    \cdot [\sigma ({a}^{2}_{1}) \cdot (1 - \sigma ({a}^{2}_{1}))]
    \cdot {w}^{2}_{21}
    \cdot [1-\tanh{({a}^{1}_{2})}]
   \cdot {1})
\end{equation}



```python
import numpy as np

class MyModel:
  def __init__(self):
    #Initialize weights with Xavier initialization
    self.weight_1 = np.random.normal(loc=0.0, scale = 1, size = (2,2))
    self.weight_2 = np.random.normal(loc=0.0, scale = 1, size = (2,1))

    #Initalize bias to zero
    self.bias_1 = np.zeros((2,1))
    self.bias_2 = np.zeros((1,1))

  def fit(self, X=None, y=None, epochs=1000, learning_rate=0.1):
    for _ in range(epochs):
      for i in range(len(X)):
        #Perform forward propagation
        yHat = self.forward(X[i].reshape((len(X[i]),1)))

        #Perform backward propagation 
        gradient_weight_1, gradient_weight_2, gradient_bias_1, gradient_bias_2 = self.backward(y[i],yHat.item())

        #Simultaneously update all parameters 
        self.weight_1 =  self.weight_1 - (learning_rate * gradient_weight_1)
        self.weight_2 = self.weight_2 - (learning_rate * gradient_weight_2)
        self.bias_1 = self.bias_1 - (learning_rate * gradient_bias_1)
        self.bias_2 = self.bias_2 - (learning_rate * gradient_bias_2)
        
  def forward(self, X):
    #Feed forward one sample at a time
    self.h_0 = X
    self.a_1 = np.dot(np.transpose(self.weight_1),X) + self.bias_1
    self.h_1 = np.tanh(self.a_1)
    self.a_2 = np.dot(np.transpose(self.weight_2),self.h_1) + self.bias_2
    self.h_2 = self.sigmoid(self.a_2)
    return self.h_2

  def backward(self, y, yHat):
    #Backward propergation one sample at a time

    #Calcualte partile derivitive
    dL_dy = self.binary_crossentropy_prime(y,yHat)
    dy_da21 = self.sigmoid_prime(self.a_2)

    #Calcualte gradients (Stacks values into matrix forms to simplify calculation)
    gradient_weight_2 =   np.full((2,1),dL_dy) * np.full((2,1),dy_da21) * self.h_1
    gradient_bias_2 =     np.full((1,1),dL_dy) * np.full((1,1),dy_da21)
    gradient_weight_1 =   np.full((2,2),dL_dy) * np.full((2,2),dy_da21) * np.transpose(np.column_stack((self.weight_2,self.weight_2))) * np.column_stack((self.tanh_prime(self.a_1),self.tanh_prime(self.a_1))).T * np.column_stack((self.h_0,self.h_0))
    gradient_bias_1 =     np.full((2,1),dL_dy) * np.full((2,1),dy_da21) * self.weight_2 * self.tanh_prime(self.a_1)

    return gradient_weight_1, gradient_weight_2, gradient_bias_1, gradient_bias_2

  def sigmoid(self,x):
      return 1/(1+np.exp(-x))

  def sigmoid_prime(self, x):
    return self.sigmoid(x) * (1.0 - self.sigmoid(x))

  def tanh(self, x ):
    return np.tanh(x)
  
  def tanh_prime(self, x):
    return 1 - np.tanh(x)**2

  def binary_crossentropy_prime(self,y, yHat):
    return -(y/yHat)+((1-y)/(1-yHat))

  def predict(self, X):
    results = np.zeros((len(X)))
    for i in range(len(X)):
       results [i] = self.forward(X[i].reshape((len(X[i]),1))).item()
    return results


X = np.array([[0,0],[0,1],[1,0],[1,1]])
y = np.array([0,1,1,0])

model = MyModel()
model.fit(X,y)

model.predict(X)

```




    array([0.01430112, 0.99191275, 0.9918608 , 0.01547358])




```python

```

5f2e0ccaa9a054aa66d983286e4268a6abdeb9c2
ipynb
Jupyter Notebook
Assignment 4/Assignment_4_keras.ipynb
sotheanith/CECS-551-Projects
f83c6f855ccceed63dfaec0d591378dabaa4bad3
MIT
Assignment 4/Assignment_4_keras.ipynb
sotheanith/CECS-551-Projects
f83c6f855ccceed63dfaec0d591378dabaa4bad3
MIT
Assignment 4/Assignment_4_keras.ipynb
sotheanith/CECS-551-Projects
f83c6f855ccceed63dfaec0d591378dabaa4bad3
MIT
Qwen/Qwen-72B
1. YES
2. YES