# 2. logistic_regression



```python
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
plt.style.use('fivethirtyeight')
import matplotlib.pyplot as plt
# import tensorflow as tf
from sklearn.metrics import classification_report
```


```python
data = pd.read_csv('ex2data1.txt', names=['exam1', 'exam2', 'admitted'])
data.head()
```




<div>
<style>
    .dataframe thead tr:only-child th {
        text-align: right;
    }

    .dataframe thead th {
        text-align: left;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>exam1</th>
      <th>exam2</th>
      <th>admitted</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>34.623660</td>
      <td>78.024693</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>30.286711</td>
      <td>43.894998</td>
      <td>0</td>
    </tr>
    <tr>
      <th>2</th>
      <td>35.847409</td>
      <td>72.902198</td>
      <td>0</td>
    </tr>
    <tr>
      <th>3</th>
      <td>60.182599</td>
      <td>86.308552</td>
      <td>1</td>
    </tr>
    <tr>
      <th>4</th>
      <td>79.032736</td>
      <td>75.344376</td>
      <td>1</td>
    </tr>
  </tbody>
</table>
</div>




```python
data.describe()
```




<div>
<style>
    .dataframe thead tr:only-child th {
        text-align: right;
    }

    .dataframe thead th {
        text-align: left;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>exam1</th>
      <th>exam2</th>
      <th>admitted</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>count</th>
      <td>100.000000</td>
      <td>100.000000</td>
      <td>100.000000</td>
    </tr>
    <tr>
      <th>mean</th>
      <td>65.644274</td>
      <td>66.221998</td>
      <td>0.600000</td>
    </tr>
    <tr>
      <th>std</th>
      <td>19.458222</td>
      <td>18.582783</td>
      <td>0.492366</td>
    </tr>
    <tr>
      <th>min</th>
      <td>30.058822</td>
      <td>30.603263</td>
      <td>0.000000</td>
    </tr>
    <tr>
      <th>25%</th>
      <td>50.919511</td>
      <td>48.179205</td>
      <td>0.000000</td>
    </tr>
    <tr>
      <th>50%</th>
      <td>67.032988</td>
      <td>67.682381</td>
      <td>1.000000</td>
    </tr>
    <tr>
      <th>75%</th>
      <td>80.212529</td>
      <td>79.360605</td>
      <td>1.000000</td>
    </tr>
    <tr>
      <th>max</th>
      <td>99.827858</td>
      <td>98.869436</td>
      <td>1.000000</td>
    </tr>
  </tbody>
</table>
</div>




```python
sns.set(context="notebook", style="darkgrid", palette=sns.color_palette("RdBu", 2))

sns.lmplot('exam1', 'exam2', hue='admitted', data=data, 
           size=6, 
           fit_reg=False, 
           scatter_kws={"s": 50}
          )
plt.show()
```


```python
def get_X(df):
#     """
#     use concat to add intersect feature to avoid side effect
#     not efficient for big dataset though
#     """
    ones = pd.DataFrame({'ones': np.ones(len(df))})
    data = pd.concat([ones, df], axis=1)  
    return data.iloc[:, :-1].as_matrix()  


def get_y(df):
#     '''assume the last column is the target'''
    return np.array(df.iloc[:, -1])#df.iloc[:, -1]


def normalize_feature(df):
#     """Applies function along input axis(default 0) of DataFrame."""
    return df.apply(lambda column: (column - column.mean()) / column.std())
```


```python
X = get_X(data)
print(X.shape)

y = get_y(data)
print(y.shape)
```

    (100, 3)
    (100,)



```python
def sigmoid(z):
    return 1 / (1 + np.exp(-z))
```


```python
fig, ax = plt.subplots(figsize=(8, 6))
ax.plot(np.arange(-10, 10, step=0.01),
        sigmoid(np.arange(-10, 10, step=0.01)))
ax.set_ylim((-0.1,1.1))
ax.set_xlabel('z', fontsize=18)
ax.set_ylabel('g(z)', fontsize=18)
ax.set_title('sigmoid function', fontsize=18)
plt.show()
```

# cost function
> * $max(\ell(\theta)) = min(-\ell(\theta))$  
> * choose $-\ell(\theta)$ as the cost function

$$\begin{align}
  & J\left( \theta  \right)=-\frac{1}{m}\sum\limits_{i=1}^{m}{[{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)+\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]} \\ 
 & =\frac{1}{m}\sum\limits_{i=1}^{m}{[-{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)-\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]} \\ 
\end{align}$$



```python
theta = theta=np.zeros(3) # X(m*n) so theta is n*1
theta
```




    array([ 0.,  0.,  0.])




```python

```


```python
def cost(theta, X, y):
    ''' cost fn is -l(theta) for you to minimize'''
    return np.mean(-y * np.log(sigmoid(X @ theta)) - (1 - y) * np.log(1 - sigmoid(X @ theta)))

# X @ thetaX.dot(theta)
```


```python
cost(theta, X, y)
```




    0.69314718055994529



# gradient descent
（batch gradient descent）  
： $\frac{1}{m} X^T( Sigmoid(X\theta) - y )$
$$\frac{\partial J\left( \theta  \right)}{\partial {{\theta }_{j}}}=\frac{1}{m}\sum\limits_{i=1}^{m}{({{h}_{\theta }}\left( {{x}^{(i)}} \right)-{{y}^{(i)}})x_{_{j}}^{(i)}}$$


```python
def gradient(theta, X, y):
#     '''just 1 batch gradient'''
    return (1 / len(X)) * X.T @ (sigmoid(X @ theta) - y)
```


```python
gradient(theta, X, y)
```




    array([ -0.1       , -12.00921659, -11.26284221])




```python
import scipy.optimize as opt
```


```python
res = opt.minimize(fun=cost, x0=theta, args=(X, y), method='Newton-CG', jac=gradient)
```


```python
print(res)
```

         fun: 0.20349770172083584
         jac: array([  1.98942032e-06,   1.34698328e-04,   1.47259166e-04])
     message: 'Optimization terminated successfully.'
        nfev: 73
        nhev: 0
         nit: 30
        njev: 270
      status: 0
     success: True
           x: array([-25.16227358,   0.20623923,   0.20147921])



```python
def predict(x, theta):
    prob = sigmoid(x @ theta)
    return (prob >= 0.5).astype(int)
```


```python
final_theta = res.x
y_pred = predict(X, final_theta)

print(classification_report(y, y_pred))
```

                 precision    recall  f1-score   support
    
              0       0.87      0.85      0.86        40
              1       0.90      0.92      0.91        60
    
    avg / total       0.89      0.89      0.89       100
    


http://stats.stackexchange.com/questions/93569/why-is-logistic-regression-a-linear-classifier
> $X \times \theta = 0$  (this is the line)


```python
print(res.x) # this is final theta
```

    [-25.16227358   0.20623923   0.20147921]



```python
coef = -(res.x / res.x[2])  # find the equation
print(coef)

x = np.arange(130, step=0.1)
y = coef[0] + coef[1]*x
```

    [ 124.88769463   -1.0236254    -1.        ]



```python
data.describe()  # find the range of x and y
```




<div>
<style>
    .dataframe thead tr:only-child th {
        text-align: right;
    }

    .dataframe thead th {
        text-align: left;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>exam1</th>
      <th>exam2</th>
      <th>admitted</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>count</th>
      <td>100.000000</td>
      <td>100.000000</td>
      <td>100.000000</td>
    </tr>
    <tr>
      <th>mean</th>
      <td>65.644274</td>
      <td>66.221998</td>
      <td>0.600000</td>
    </tr>
    <tr>
      <th>std</th>
      <td>19.458222</td>
      <td>18.582783</td>
      <td>0.492366</td>
    </tr>
    <tr>
      <th>min</th>
      <td>30.058822</td>
      <td>30.603263</td>
      <td>0.000000</td>
    </tr>
    <tr>
      <th>25%</th>
      <td>50.919511</td>
      <td>48.179205</td>
      <td>0.000000</td>
    </tr>
    <tr>
      <th>50%</th>
      <td>67.032988</td>
      <td>67.682381</td>
      <td>1.000000</td>
    </tr>
    <tr>
      <th>75%</th>
      <td>80.212529</td>
      <td>79.360605</td>
      <td>1.000000</td>
    </tr>
    <tr>
      <th>max</th>
      <td>99.827858</td>
      <td>98.869436</td>
      <td>1.000000</td>
    </tr>
  </tbody>
</table>
</div>



> you know the intercept would be around 125 for both x and y


```python
sns.set(context="notebook", style="ticks", font_scale=1.5)

sns.lmplot('exam1', 'exam2', hue='admitted', data=data, 
           size=6, 
           fit_reg=False, 
           scatter_kws={"s": 25}
          )

plt.plot(x, y, 'grey')
plt.xlim(0, 130)
plt.ylim(0, 130)
plt.title('Decision Boundary')
plt.show()
```

# 3- 正则化逻辑回归


```python
df = pd.read_csv('ex2data2.txt', names=['test1', 'test2', 'accepted'])
df.head()
```




<div>
<style>
    .dataframe thead tr:only-child th {
        text-align: right;
    }

    .dataframe thead th {
        text-align: left;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>test1</th>
      <th>test2</th>
      <th>accepted</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0.051267</td>
      <td>0.69956</td>
      <td>1</td>
    </tr>
    <tr>
      <th>1</th>
      <td>-0.092742</td>
      <td>0.68494</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>-0.213710</td>
      <td>0.69225</td>
      <td>1</td>
    </tr>
    <tr>
      <th>3</th>
      <td>-0.375000</td>
      <td>0.50219</td>
      <td>1</td>
    </tr>
    <tr>
      <th>4</th>
      <td>-0.513250</td>
      <td>0.46564</td>
      <td>1</td>
    </tr>
  </tbody>
</table>
</div>




```python
sns.set(context="notebook", style="ticks", font_scale=1.5)

sns.lmplot('test1', 'test2', hue='accepted', data=df, 
           size=6, 
           fit_reg=False, 
           scatter_kws={"s": 50}
          )

plt.title('Regularized Logistic Regression')
plt.show()
```

# feature mapping（特征映射）

polynomial expansion

```
for i in 0..i
  for p in 0..i:
    output x^(i-p) * y^p
```



```python
def feature_mapping(x, y, power, as_ndarray=False):
#     """return mapped features as ndarray or dataframe"""
    # data = {}
    # # inclusive
    # for i in np.arange(power + 1):
    #     for p in np.arange(i + 1):
    #         data["f{}{}".format(i - p, p)] = np.power(x, i - p) * np.power(y, p)

    data = {"f{}{}".format(i - p, p): np.power(x, i - p) * np.power(y, p)
                for i in np.arange(power + 1)
                for p in np.arange(i + 1)
            }

    if as_ndarray:
        return pd.DataFrame(data).as_matrix()
    else:
        return pd.DataFrame(data)

```


```python
x1 = np.array(df.test1)
x2 = np.array(df.test2)
```


```python
data = feature_mapping(x1, x2, power=6)
print(data.shape)
data.head()
```

    (118, 28)





<div>
<style>
    .dataframe thead tr:only-child th {
        text-align: right;
    }

    .dataframe thead th {
        text-align: left;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>f00</th>
      <th>f01</th>
      <th>f02</th>
      <th>f03</th>
      <th>f04</th>
      <th>f05</th>
      <th>f06</th>
      <th>f10</th>
      <th>f11</th>
      <th>f12</th>
      <th>...</th>
      <th>f30</th>
      <th>f31</th>
      <th>f32</th>
      <th>f33</th>
      <th>f40</th>
      <th>f41</th>
      <th>f42</th>
      <th>f50</th>
      <th>f51</th>
      <th>f60</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1.0</td>
      <td>0.69956</td>
      <td>0.489384</td>
      <td>0.342354</td>
      <td>0.239497</td>
      <td>0.167542</td>
      <td>0.117206</td>
      <td>0.051267</td>
      <td>0.035864</td>
      <td>0.025089</td>
      <td>...</td>
      <td>0.000135</td>
      <td>0.000094</td>
      <td>0.000066</td>
      <td>0.000046</td>
      <td>0.000007</td>
      <td>0.000005</td>
      <td>0.000003</td>
      <td>3.541519e-07</td>
      <td>2.477505e-07</td>
      <td>1.815630e-08</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1.0</td>
      <td>0.68494</td>
      <td>0.469143</td>
      <td>0.321335</td>
      <td>0.220095</td>
      <td>0.150752</td>
      <td>0.103256</td>
      <td>-0.092742</td>
      <td>-0.063523</td>
      <td>-0.043509</td>
      <td>...</td>
      <td>-0.000798</td>
      <td>-0.000546</td>
      <td>-0.000374</td>
      <td>-0.000256</td>
      <td>0.000074</td>
      <td>0.000051</td>
      <td>0.000035</td>
      <td>-6.860919e-06</td>
      <td>-4.699318e-06</td>
      <td>6.362953e-07</td>
    </tr>
    <tr>
      <th>2</th>
      <td>1.0</td>
      <td>0.69225</td>
      <td>0.479210</td>
      <td>0.331733</td>
      <td>0.229642</td>
      <td>0.158970</td>
      <td>0.110047</td>
      <td>-0.213710</td>
      <td>-0.147941</td>
      <td>-0.102412</td>
      <td>...</td>
      <td>-0.009761</td>
      <td>-0.006757</td>
      <td>-0.004677</td>
      <td>-0.003238</td>
      <td>0.002086</td>
      <td>0.001444</td>
      <td>0.001000</td>
      <td>-4.457837e-04</td>
      <td>-3.085938e-04</td>
      <td>9.526844e-05</td>
    </tr>
    <tr>
      <th>3</th>
      <td>1.0</td>
      <td>0.50219</td>
      <td>0.252195</td>
      <td>0.126650</td>
      <td>0.063602</td>
      <td>0.031940</td>
      <td>0.016040</td>
      <td>-0.375000</td>
      <td>-0.188321</td>
      <td>-0.094573</td>
      <td>...</td>
      <td>-0.052734</td>
      <td>-0.026483</td>
      <td>-0.013299</td>
      <td>-0.006679</td>
      <td>0.019775</td>
      <td>0.009931</td>
      <td>0.004987</td>
      <td>-7.415771e-03</td>
      <td>-3.724126e-03</td>
      <td>2.780914e-03</td>
    </tr>
    <tr>
      <th>4</th>
      <td>1.0</td>
      <td>0.46564</td>
      <td>0.216821</td>
      <td>0.100960</td>
      <td>0.047011</td>
      <td>0.021890</td>
      <td>0.010193</td>
      <td>-0.513250</td>
      <td>-0.238990</td>
      <td>-0.111283</td>
      <td>...</td>
      <td>-0.135203</td>
      <td>-0.062956</td>
      <td>-0.029315</td>
      <td>-0.013650</td>
      <td>0.069393</td>
      <td>0.032312</td>
      <td>0.015046</td>
      <td>-3.561597e-02</td>
      <td>-1.658422e-02</td>
      <td>1.827990e-02</td>
    </tr>
  </tbody>
</table>
<p>5 rows × 28 columns</p>
</div>




```python
data.describe()
```




<div>
<style>
    .dataframe thead tr:only-child th {
        text-align: right;
    }

    .dataframe thead th {
        text-align: left;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>f00</th>
      <th>f01</th>
      <th>f02</th>
      <th>f03</th>
      <th>f04</th>
      <th>f05</th>
      <th>f06</th>
      <th>f10</th>
      <th>f11</th>
      <th>f12</th>
      <th>...</th>
      <th>f30</th>
      <th>f31</th>
      <th>f32</th>
      <th>f33</th>
      <th>f40</th>
      <th>f41</th>
      <th>f42</th>
      <th>f50</th>
      <th>f51</th>
      <th>f60</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>count</th>
      <td>118.0</td>
      <td>118.000000</td>
      <td>118.000000</td>
      <td>118.000000</td>
      <td>1.180000e+02</td>
      <td>118.000000</td>
      <td>1.180000e+02</td>
      <td>118.000000</td>
      <td>118.000000</td>
      <td>118.000000</td>
      <td>...</td>
      <td>1.180000e+02</td>
      <td>118.000000</td>
      <td>1.180000e+02</td>
      <td>118.000000</td>
      <td>1.180000e+02</td>
      <td>118.000000</td>
      <td>1.180000e+02</td>
      <td>1.180000e+02</td>
      <td>118.000000</td>
      <td>1.180000e+02</td>
    </tr>
    <tr>
      <th>mean</th>
      <td>1.0</td>
      <td>0.183102</td>
      <td>0.301370</td>
      <td>0.142350</td>
      <td>1.710985e-01</td>
      <td>0.115710</td>
      <td>1.257256e-01</td>
      <td>0.054779</td>
      <td>-0.025472</td>
      <td>0.015483</td>
      <td>...</td>
      <td>5.983333e-02</td>
      <td>-0.005251</td>
      <td>9.432094e-03</td>
      <td>-0.001705</td>
      <td>1.225384e-01</td>
      <td>0.011812</td>
      <td>1.893340e-02</td>
      <td>5.196507e-02</td>
      <td>-0.000703</td>
      <td>7.837118e-02</td>
    </tr>
    <tr>
      <th>std</th>
      <td>0.0</td>
      <td>0.519743</td>
      <td>0.284536</td>
      <td>0.326134</td>
      <td>2.815658e-01</td>
      <td>0.299092</td>
      <td>2.964416e-01</td>
      <td>0.496654</td>
      <td>0.224075</td>
      <td>0.150143</td>
      <td>...</td>
      <td>2.746459e-01</td>
      <td>0.096738</td>
      <td>5.455787e-02</td>
      <td>0.037443</td>
      <td>2.092709e-01</td>
      <td>0.072274</td>
      <td>3.430092e-02</td>
      <td>2.148098e-01</td>
      <td>0.058271</td>
      <td>1.938621e-01</td>
    </tr>
    <tr>
      <th>min</th>
      <td>1.0</td>
      <td>-0.769740</td>
      <td>0.000026</td>
      <td>-0.456071</td>
      <td>6.855856e-10</td>
      <td>-0.270222</td>
      <td>1.795116e-14</td>
      <td>-0.830070</td>
      <td>-0.484096</td>
      <td>-0.483743</td>
      <td>...</td>
      <td>-5.719317e-01</td>
      <td>-0.296854</td>
      <td>-1.592528e-01</td>
      <td>-0.113448</td>
      <td>1.612020e-09</td>
      <td>-0.246068</td>
      <td>2.577297e-10</td>
      <td>-3.940702e-01</td>
      <td>-0.203971</td>
      <td>6.472253e-14</td>
    </tr>
    <tr>
      <th>25%</th>
      <td>1.0</td>
      <td>-0.254385</td>
      <td>0.061086</td>
      <td>-0.016492</td>
      <td>3.741593e-03</td>
      <td>-0.001072</td>
      <td>2.298277e-04</td>
      <td>-0.372120</td>
      <td>-0.178209</td>
      <td>-0.042980</td>
      <td>...</td>
      <td>-5.155632e-02</td>
      <td>-0.029360</td>
      <td>-3.659760e-03</td>
      <td>-0.005749</td>
      <td>1.869975e-03</td>
      <td>-0.001926</td>
      <td>1.258285e-04</td>
      <td>-7.147973e-03</td>
      <td>-0.006381</td>
      <td>8.086369e-05</td>
    </tr>
    <tr>
      <th>50%</th>
      <td>1.0</td>
      <td>0.213455</td>
      <td>0.252195</td>
      <td>0.009734</td>
      <td>6.360222e-02</td>
      <td>0.000444</td>
      <td>1.604015e-02</td>
      <td>-0.006336</td>
      <td>-0.016521</td>
      <td>-0.000039</td>
      <td>...</td>
      <td>-2.544062e-07</td>
      <td>-0.000512</td>
      <td>-1.473547e-07</td>
      <td>-0.000005</td>
      <td>2.736163e-02</td>
      <td>0.000205</td>
      <td>3.387050e-03</td>
      <td>-1.021440e-11</td>
      <td>-0.000004</td>
      <td>4.527344e-03</td>
    </tr>
    <tr>
      <th>75%</th>
      <td>1.0</td>
      <td>0.646562</td>
      <td>0.464189</td>
      <td>0.270310</td>
      <td>2.155453e-01</td>
      <td>0.113020</td>
      <td>1.001215e-01</td>
      <td>0.478970</td>
      <td>0.100795</td>
      <td>0.079510</td>
      <td>...</td>
      <td>1.099616e-01</td>
      <td>0.015050</td>
      <td>1.370560e-02</td>
      <td>0.001024</td>
      <td>1.520801e-01</td>
      <td>0.019183</td>
      <td>2.090875e-02</td>
      <td>2.526861e-02</td>
      <td>0.002104</td>
      <td>5.932959e-02</td>
    </tr>
    <tr>
      <th>max</th>
      <td>1.0</td>
      <td>1.108900</td>
      <td>1.229659</td>
      <td>1.363569</td>
      <td>1.512062e+00</td>
      <td>1.676725</td>
      <td>1.859321e+00</td>
      <td>1.070900</td>
      <td>0.568307</td>
      <td>0.505577</td>
      <td>...</td>
      <td>1.228137e+00</td>
      <td>0.369805</td>
      <td>2.451845e-01</td>
      <td>0.183548</td>
      <td>1.315212e+00</td>
      <td>0.304409</td>
      <td>2.018260e-01</td>
      <td>1.408460e+00</td>
      <td>0.250577</td>
      <td>1.508320e+00</td>
    </tr>
  </tbody>
</table>
<p>8 rows × 28 columns</p>
</div>



# regularized cost

$$J\left( \theta  \right)=\frac{1}{m}\sum\limits_{i=1}^{m}{[-{{y}^{(i)}}\log \left( {{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)-\left( 1-{{y}^{(i)}} \right)\log \left( 1-{{h}_{\theta }}\left( {{x}^{(i)}} \right) \right)]}+\frac{\lambda }{2m}\sum\limits_{j=1}^{n}{\theta _{j}^{2}}$$


```python
theta = np.zeros(data.shape[1])
X = feature_mapping(x1, x2, power=6, as_ndarray=True)
print(X.shape)

y = get_y(df)
print(y.shape)
```

    (118, 28)
    (118,)



```python
def regularized_cost(theta, X, y, l=1):
#     '''you don't penalize theta_0'''
    theta_j1_to_n = theta[1:]
    regularized_term = (l / (2 * len(X))) * np.power(theta_j1_to_n, 2).sum()

    return cost(theta, X, y) + regularized_term
#正则化代价函数
```


```python
regularized_cost(theta, X, y, l=1)
```




    0.6931471805599454



this is the same as the not regularized cost because we init theta as zeros...

# regularized gradient
$$\frac{\partial J\left( \theta  \right)}{\partial {{\theta }_{j}}}=\left( \frac{1}{m}\sum\limits_{i=1}^{m}{\left( {{h}_{\theta }}\left( {{x}^{\left( i \right)}} \right)-{{y}^{\left( i \right)}} \right)} \right)+\frac{\lambda }{m}{{\theta }_{j}}\text{ }\text{             for  j}\ge \text{1}$$


```python
def regularized_gradient(theta, X, y, l=1):
#     '''still, leave theta_0 alone'''
    theta_j1_to_n = theta[1:]
    regularized_theta = (l / len(X)) * theta_j1_to_n

    # by doing this, no offset is on theta_0
    regularized_term = np.concatenate([np.array([0]), regularized_theta])

    return gradient(theta, X, y) + regularized_term
```


```python
regularized_gradient(theta, X, y)
```




    array([  8.47457627e-03,   7.77711864e-05,   3.76648474e-02,
             2.34764889e-02,   3.93028171e-02,   3.10079849e-02,
             3.87936363e-02,   1.87880932e-02,   1.15013308e-02,
             8.19244468e-03,   3.09593720e-03,   4.47629067e-03,
             1.37646175e-03,   5.03446395e-02,   7.32393391e-03,
             1.28600503e-02,   5.83822078e-03,   7.26504316e-03,
             1.83559872e-02,   2.23923907e-03,   3.38643902e-03,
             4.08503006e-04,   3.93486234e-02,   4.32983232e-03,
             6.31570797e-03,   1.99707467e-02,   1.09740238e-03,
             3.10312442e-02])




```python
import scipy.optimize as opt
```


```python
print('init cost = {}'.format(regularized_cost(theta, X, y)))

res = opt.minimize(fun=regularized_cost, x0=theta, args=(X, y), method='Newton-CG', jac=regularized_gradient)
res
```

    init cost = 0.6931471805599454





         fun: 0.5290027297127737
         jac: array([  1.05541650e-07,   6.13738958e-08,  -7.04772465e-08,
            -1.36777274e-08,  -2.84652117e-08,  -3.41659162e-08,
            -5.15630062e-08,  -1.74645557e-08,   7.50726574e-09,
             1.18564041e-08,   1.78853446e-08,   1.12436692e-08,
             8.73092759e-09,   5.56139873e-08,   4.12686771e-09,
            -2.49410770e-08,  -6.34978298e-09,  -1.34271390e-08,
            -2.13487848e-09,   4.54710087e-10,  -4.37439525e-09,
            -1.26745782e-09,  -3.40985521e-09,   7.34158338e-09,
            -7.74561697e-09,  -9.84723852e-11,   9.65250750e-10,
            -1.18501368e-08])
     message: 'Optimization terminated successfully.'
        nfev: 7
        nhev: 0
         nit: 6
        njev: 66
      status: 0
     success: True
           x: array([ 1.27273981,  1.18108974, -1.43166669, -0.17513036, -1.19281478,
           -0.45635758, -0.9246528 ,  0.62527237, -0.9174247 , -0.35723884,
           -0.27470605, -0.29537769, -0.14388711, -2.0199599 , -0.36553508,
           -0.61555685, -0.27778507, -0.32738029,  0.12400668, -0.05098942,
           -0.04473108,  0.01556645, -1.45815829, -0.20600596, -0.29243192,
           -0.24218804,  0.02777165, -1.04320421])




```python
final_theta = res.x
y_pred = predict(X, final_theta)

print(classification_report(y, y_pred))
```

                 precision    recall  f1-score   support
    
              0       0.90      0.75      0.82        60
              1       0.78      0.91      0.84        58
    
    avg / total       0.84      0.83      0.83       118
    


#  $\lambda$
# 
*  $X\times \theta = 0$ 
* instead of solving polynomial equation, just create a coridate x,y grid that is dense enough, and find all those $X\times \theta$ that is close enough to 0, then plot them


```python
def draw_boundary(power, l):
#     """
#     power: polynomial power for mapped feature
#     l: lambda constant
#     """
    density = 1000
    threshhold = 2 * 10**-3

    final_theta = feature_mapped_logistic_regression(power, l)
    x, y = find_decision_boundary(density, power, final_theta, threshhold)

    df = pd.read_csv('ex2data2.txt', names=['test1', 'test2', 'accepted'])
    sns.lmplot('test1', 'test2', hue='accepted', data=df, size=6, fit_reg=False, scatter_kws={"s": 100})

    plt.scatter(x, y, c='R', s=10)
    plt.title('Decision boundary')
    plt.show()
```


```python
def feature_mapped_logistic_regression(power, l):
#     """for drawing purpose only.. not a well generealize logistic regression
#     power: int
#         raise x1, x2 to polynomial power
#     l: int
#         lambda constant for regularization term
#     """
    df = pd.read_csv('ex2data2.txt', names=['test1', 'test2', 'accepted'])
    x1 = np.array(df.test1)
    x2 = np.array(df.test2)
    y = get_y(df)

    X = feature_mapping(x1, x2, power, as_ndarray=True)
    theta = np.zeros(X.shape[1])

    res = opt.minimize(fun=regularized_cost,
                       x0=theta,
                       args=(X, y, l),
                       method='TNC',
                       jac=regularized_gradient)
    final_theta = res.x

    return final_theta
```


```python
def find_decision_boundary(density, power, theta, threshhold):
    t1 = np.linspace(-1, 1.5, density)
    t2 = np.linspace(-1, 1.5, density)

    cordinates = [(x, y) for x in t1 for y in t2]
    x_cord, y_cord = zip(*cordinates)
    mapped_cord = feature_mapping(x_cord, y_cord, power)  # this is a dataframe

    inner_product = mapped_cord.as_matrix() @ theta

    decision = mapped_cord[np.abs(inner_product) < threshhold]

    return decision.f10, decision.f01
```


```python
draw_boundary(power=6, l=1)#lambda=1
```


```python
draw_boundary(power=6, l=0)  # no regularization, over fitting，#lambda=0
```


```python
draw_boundary(power=6, l=100)  # underfitting，#lambda=100
```


```python

```

44363ec14735d42b6b7cf3b4e02d38e0b41fdd22
ipynb
Jupyter Notebook
Python/2-Logistic Regression/.ipynb_checkpoints/1. logistic_regression_v1-checkpoint.ipynb
jayskhatri/explore-ml-workshop
0a2120507bd60f3f31350bc628dcee71dcf9d11a
MIT
2020-01-04T08:40:13.000Z
2020-01-04T10:18:46.000Z
Python/2-Logistic Regression/.ipynb_checkpoints/1. logistic_regression_v1-checkpoint.ipynb
jayskhatri/explore-ml-workshop
0a2120507bd60f3f31350bc628dcee71dcf9d11a
MIT
Python/2-Logistic Regression/.ipynb_checkpoints/1. logistic_regression_v1-checkpoint.ipynb
jayskhatri/explore-ml-workshop
0a2120507bd60f3f31350bc628dcee71dcf9d11a
MIT
2020-01-04T07:24:06.000Z
2020-01-04T10:06:15.000Z
Qwen/Qwen-72B
1. YES
2. YES