```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
```


```python
# Ensure that we always get the same results
np.random.seed(0)

# Helper function to generate a random data sample
def generate_random_data_sample(sample_size, feature_dim=2, num_classes=2):
    # Create synthetic data using NumPy.
    Y = np.random.randint(size=(sample_size, 1), low=0, high=num_classes)
    # Make sure that the data is separable
    X = (np.random.randn(sample_size, feature_dim)+5) * (Y+1)
    return X, Y

# Helper function to draw decision boundary
def draw_result(x, x_values, y_values, cost=None):
    plt.figure(figsize=(10,5))
    ax = plt.gca()
    ax.grid(color='#b7b7b7', linestyle='-', linewidth=0.5, alpha=0.5)
    plt.scatter(x[:,0], x[:,1], c=colors)
    plt.plot(x_values, y_values, label='Decision Boundary', color='#121212', linewidth=2, alpha=0.9)
    if cost:
        ax.text(0, 0, f'error = {cost:.2f}',fontsize=12,color='#000000')
    plt.show()
```


```python
# Generate random separable data
sample_size = 128
x, labels = generate_random_data_sample(sample_size)
y = np.array([labels.T[0]]).T
colors = ['#333333' if label == 0 else '#b7b7b7' for label in labels]

# generate random decision boundary
theta = np.random.randn(3)
x_values = [np.min(x[:, 0] - 5), np.max(x[:, 1] + 5)]
y_values = -(theta[0] + np.dot(theta[1], x_values))/theta[2]

draw_result(x, x_values, y_values)
```

## Logistic regression

Optimize decision boundary with logistic regression.

### 1. Feedforward
\begin{equation}
\mathbf{z} = XW
\end{equation}

\begin{equation}
\mathbf{\hat{y}} = \sigma(\mathbf{z})
\end{equation}

\begin{equation}
\sigma(z) = \frac{1}{1+e^{-z}}
\end{equation}

### 2. Compute cost function
\begin{equation}
L(\mathbf{y}, \mathbf{\hat{y}}) = -\frac{1}{m}\big(\mathbf{y}^T\log(\mathbf{\hat{y}})+(1-\mathbf{y})^T\log(1-\mathbf{\hat{y}})\big)
\end{equation}

### 3. Backpropagation
\begin{equation}
\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial W} 
= \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial \mathbf{\hat{y}}} \cdot \frac{\partial \mathbf{\hat{y}}}{\partial W} 
= \frac{1}{m}X^T(\mathbf{\hat{y}} - \mathbf{y})
\end{equation}

### 4. Gradient descent
\begin{equation}
W = W - \alpha  \frac{\delta Loss(y, \hat{y})}{\delta W}
\end{equation}


```python
class LogisticRegression:
    """ Simple logistic regression """
    def __init__(self):
        pass
    
    def _init_params(self, x, y, iterations, learning_rate, reg_factor):
        """ Initilize parameters. 
        
        ----------
        W : ndarray, shape (n_features+1,)
            Coefficient vector
        """
        self._X = np.hstack([np.ones((x.shape[0], 1)), x])
        self._y = y
        self._learning_rate = learning_rate
        self._reg_factor = reg_factor
        self.weights_ = np.random.rand(self._X.shape[1],1)
        self.costs_ = np.zeros(iterations)
        
    def _sigmoid(self, x):
        """ Computes sigmoid function. """
        return 1.0/(1 + np.exp(-x))
    
    def _feedforward(self):
        """ Computes np.dot(X, W). """
        self._y_hat = self._sigmoid(self._X.dot(self.weights_))
        
    def _backprop(self):
        """ Update weights. """
        m = len(self._y)
        # update weights with L2 regularization term
        _weights = self.weights_.copy()
        # ignore bias term
        _weights[0, 0] = 0 
        self.weights_ -= self._learning_rate * self._X.T.dot(self._y_hat - self._y)/m + self._reg_factor/m*_weights
    
    def _get_cost(self):
        """ Compute loss. """
        m = len(self._y)
        # cost function with L2 regularization term
        _weights = self.weights_.copy()
        # ignore bias term
        _weights[0, 0] = 0    
        return -1/m * (self._y.T.dot(np.log(self._y_hat)) + (1-self._y).T.dot(np.log(1-self._y_hat))) + self._reg_factor/(2*m)*_weights.T.dot(_weights)

    def fit(self, x, y, iterations=1000, learning_rate=0.2, reg_factor=0.5):
        """ Fit model.
        
        ----------
        x : ndarray, shape (n_samples, n_features)
            Training data
        y : ndarray, shape (n_samples,)
            Target data
        """
        self._init_params(x, y, iterations, learning_rate, reg_factor)
        
        # train model
        for i in range(iterations):
            self._feedforward()
            self._backprop()
            self.costs_[i] = self._get_cost()
```


```python
lr = LogisticRegression()
lr.fit(x, y)

new_y_values = -(lr.weights_[0] + lr.weights_[1]* x_values)/lr.weights_[2]

draw_result(x, x_values, new_y_values, lr.costs_[-1])
```

7ba501003de4e96b26269d8d8af5cb2fc89329a7
ipynb
Jupyter Notebook
006_logistic_regression.ipynb
ValleyZw/hill
51d4bc76d7c49b3dccf88659fade806906a9b4e2
MIT
2019-07-22T07:19:39.000Z
2019-07-22T07:19:39.000Z
006_logistic_regression.ipynb
ValleyZw/hill
51d4bc76d7c49b3dccf88659fade806906a9b4e2
MIT
006_logistic_regression.ipynb
ValleyZw/hill
51d4bc76d7c49b3dccf88659fade806906a9b4e2
MIT
2020-05-19T17:52:16.000Z
2020-05-19T17:52:16.000Z
Qwen/Qwen-72B
1. YES
2. YES