```python
import numpy as np
```


<div style="width:image width px; font-size:100%; text-align:center;"> Figure 1 </div>

# ReLU 
\begin{align}
 \mathbf{z_1} &= \mathbf{W_1}\mathbf{x} + \mathbf{b_1}  \tag{1} \\
 \mathbf{h} &= \mathrm{ReLU}(\mathbf{z_1})  \tag{2} \\
\end{align}


```python
np.random.seed(10)
```


```python
z1= 10 * np.random.rand(5,1) - 5
```


```python
z1
```




    array([[ 2.71320643],
           [-4.79248051],
           [ 1.33648235],
           [ 2.48803883],
           [-0.01492988]])




```python
h = z1.copy()
```


```python
h<0
```




    array([[False],
           [ True],
           [False],
           [False],
           [ True]])




```python
h[h<0] = 0
```


```python
h
```




    array([[2.71320643],
           [0.        ],
           [1.33648235],
           [2.48803883],
           [0.        ]])




```python
np.max(h,0)
```




    array([2.71320643])




```python
np.array([max(i,0) for i in h])
```




    array([[2.71320643],
           [0.        ],
           [1.33648235],
           [2.48803883],
           [0.        ]])




```python
for i in h:
    if i<0:
        h[i] = 0
h

        
```




    array([[2.71320643],
           [0.        ],
           [1.33648235],
           [2.48803883],
           [0.        ]])




```python
def relu(z):
    result = z.copy()
    result[result<0]=0
    return result
    
```


```python
z1
```




    array([[ 2.71320643],
           [-4.79248051],
           [ 1.33648235],
           [ 2.48803883],
           [-0.01492988]])




```python
relu(z1)
```




    array([[2.71320643],
           [0.        ],
           [1.33648235],
           [2.48803883],
           [0.        ]])



# Softmax

\begin{align}
 \mathbf{z_2} &= \mathbf{W_2}\mathbf{h} + \mathbf{b_2}   \tag{3} \\
 \mathbf{\hat y} &= \mathrm{softmax}(\mathbf{z_2})   \tag{4} \\
\end{align}


$$ \textrm{softmax}(\textbf{z})_i = \frac{e^{z_i} }{\sum\limits_{j=1}^{V} e^{z_j} }  \tag{5} $$




```python
z=np.array([9,8,11,10,8.5])
```


```python
z
```




    array([ 9. ,  8. , 11. , 10. ,  8.5])




```python
np.exp(z)
```




    array([ 8103.08392758,  2980.95798704, 59874.1417152 , 22026.46579481,
            4914.7688403 ])




```python
np.sum(np.exp(z))
```




    97899.41826492078




```python
np.exp(z)/np.sum(np.exp(z))
```




    array([0.08276948, 0.03044919, 0.61158833, 0.22499077, 0.05020223])




```python
def softmax(z):
    e_z= np.exp(z)
    sum_ez = np.sum(e_z)
    return e_z / sum_ez
```


```python
softmax(z)
```




    array([0.08276948, 0.03044919, 0.61158833, 0.22499077, 0.05020223])




```python

```

b8234418c16fa5d6b22e4d3c34e2f638fdc7b9cc
ipynb
Jupyter Notebook
Module_II/Notebooks_Clase/activation_functions.ipynb
edgmz28/NLP_Course
c7638828727a685b4601b0f96d4990b94ba6820a
MIT
Module_II/Notebooks_Clase/activation_functions.ipynb
edgmz28/NLP_Course
c7638828727a685b4601b0f96d4990b94ba6820a
MIT
Module_II/Notebooks_Clase/activation_functions.ipynb
edgmz28/NLP_Course
c7638828727a685b4601b0f96d4990b94ba6820a
MIT
2021-07-17T00:37:25.000Z
2021-08-07T15:15:50.000Z
Qwen/Qwen-72B
1. YES
2. YES