# Visualization of data


```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
```


```python
file_name = r'data/data.csv'
df = pd.read_csv(file_name, engine='python')
df[['R&D Spend', 'Administration', 'Marketing Spend', 'Profit']] /= 1000
df.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>R&amp;D Spend</th>
      <th>Administration</th>
      <th>Marketing Spend</th>
      <th>State</th>
      <th>Profit</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>165.34920</td>
      <td>136.89780</td>
      <td>471.78410</td>
      <td>New York</td>
      <td>192.26183</td>
    </tr>
    <tr>
      <th>1</th>
      <td>162.59770</td>
      <td>151.37759</td>
      <td>443.89853</td>
      <td>California</td>
      <td>191.79206</td>
    </tr>
    <tr>
      <th>2</th>
      <td>153.44151</td>
      <td>101.14555</td>
      <td>407.93454</td>
      <td>Florida</td>
      <td>191.05039</td>
    </tr>
    <tr>
      <th>3</th>
      <td>144.37241</td>
      <td>118.67185</td>
      <td>383.19962</td>
      <td>New York</td>
      <td>182.90199</td>
    </tr>
    <tr>
      <th>4</th>
      <td>142.10734</td>
      <td>91.39177</td>
      <td>366.16842</td>
      <td>Florida</td>
      <td>166.18794</td>
    </tr>
  </tbody>
</table>
</div>




```python
fig = plt.figure(figsize=(6, 6))

plt.matshow(df.corr(), fignum=fig.number)
cb = plt.colorbar()
plt.title('Correlation Matrix')

plt.show()
```


```python
fig = plt.figure(figsize=(24, 6))

ax = fig.add_subplot(131)
ax.scatter(df['R&D Spend'], df['Profit'], c='r', s=20, marker='o')
ax.set_xlabel('R&D Spend')
ax.set_ylabel('Profit')

ax = fig.add_subplot(132)
ax.scatter(df['Marketing Spend'], df['Profit'], c='b', s=20, marker='o')
ax.set_xlabel('Marketing Spend')
ax.set_ylabel('Profit')

ax = fig.add_subplot(133, projection='3d')
ax.scatter(df['R&D Spend'], df['Marketing Spend'], df['Profit'], s=50, marker='o')
ax.set_xlabel('R&D Spend')
ax.set_ylabel('Marketing Spend')
ax.set_zlabel('Profit')

plt.show()
```

# Loss function
\begin{equation}
J(\theta) = \frac{1}{2}\sum_{i=1}^{n} \left(h_{\theta}(x^{i}) - y^{i} \right)^{2}
\end{equation}


```python
def loss(X, y, theta):
    y_hat = np.dot(X, theta)
    J = 0.5*sum(np.power(y-y_hat, 2))
    return J
```

# Batch gradient descent algorithm
Repeat until convergence:
\begin{equation}
\theta_{j} := \theta_{j} + \alpha \sum_{i=1}^{n}\left(y^{i} - h_{\theta}(x^{i}) \right)x_{j}^{i}
\end{equation}


```python
def batch_gradient_descent(X, y, theta, alpha, tolerance):
    old_theta = []
    iterations = []
    num = 0
    
    while 1:
        num += 1
        if num % 1000 == 0:
            iterations.append(num)
            old_theta.append(theta)
        
        y_hat = np.dot(X, theta)
        new_theta = theta + alpha*np.dot(X.T, y-y_hat) 
        if (abs(new_theta-theta) < tolerance).all():
            break
        theta = new_theta
    
    return theta, iterations, old_theta
```


```python
init_theta = np.matrix((0, 0)).T
tolerance = 1e-5
learning_rate = 5e-6

y = np.matrix([df['Profit'].values]).T
x = np.matrix([np.ones(len(y)), df['R&D Spend'].values]).T
theta, iterations, old_theta = batch_gradient_descent(x, y, init_theta, learning_rate, tolerance)
pred = np.dot(x, theta)
print(theta)

J = np.zeros(len(old_theta))
for i, value in enumerate(old_theta):
    J[i] = loss(x, y, value)

fig = plt.figure(figsize=(16, 6))

ax = fig.add_subplot(121)
ax.scatter([x[:, 1]], [y], c='b', marker='x')
ax.scatter([x[:, 1]], [pred], c='r', marker='o')
ax.set_xlabel('R&D Spend')
ax.set_ylabel('Profit')

ax = fig.add_subplot(122)
ax.plot(iterations, J, 'bo')
ax.set_xlabel('Iterations')
ax.set_ylabel('Loss function')

plt.show()
```


```python
init_theta = np.matrix((0, 0, 0)).T
tolerance = 1e-5
learning_rate = 5e-6

y = np.matrix([df['Profit'].values]).T
x = np.matrix([np.ones(len(y)), df['R&D Spend'].values, df['Marketing Spend'].values]).T
x = StandardScaler().fit_transform(x)
theta, iterations, old_theta = batch_gradient_descent(x, y, init_theta, learning_rate, tolerance)
pred = np.dot(x, theta)
print(theta)

J = np.zeros(len(old_theta))
for i, value in enumerate(old_theta):
    J[i] = loss(x, y, value)

fig = plt.figure(figsize=(16, 6))

ax = fig.add_subplot(121, projection='3d')
ax.scatter(x[:, 1], x[:, 2], y, c='b', s=50, marker='x')
ax.scatter(x[:, 1], x[:, 2], pred, c='r', s=50, marker='o')
ax.set_xlabel('Normalized R&D Spend')
ax.set_ylabel('Normalized Marketing Spend')
ax.set_zlabel('Profit')

ax = fig.add_subplot(122)
ax.plot(iterations, J, 'bo')
ax.set_xlabel('Iterations')
ax.set_ylabel('Loss function')

plt.show()
```

Batch gradient descent algorithm can't be used to find a global minimum of the loss function. It converges to a local minimum but the predicted values $h_{\theta}(x^{i})$ are far from the real values $y^{i}$. It is necessary to choose other initial coefficients $\theta_{j}$.

# Stochastic gradient descent algorithm
For $i=1$ to $n$:
\begin{equation}
\theta_{j} := \theta_{j} + \alpha\left(y^{i}-h_{\theta}(x^{i}) \right)x_{j}^{i}
\end{equation}


```python
def stochastic_gradient_descent(X, y, theta, alpha, num):
    old_theta = []
    iterations = []
    
    for i in range(num):
        if i % 1000 == 0:
            iterations.append(i)
            old_theta.append(theta)
        
        y_hat = np.dot(X, theta)
        theta = theta - alpha*np.dot(X.T, y_hat-y)
    
    return theta, old_theta, iterations
```


```python
init_theta = np.matrix((100, 0)).T
learning_rate = 5e-6
num = 100000

y = np.matrix(df['Profit'].values).T
x = np.matrix([np.ones(len(y)), df['R&D Spend'].values]).T
theta, old_theta, iterations = stochastic_gradient_descent(x, y, init_theta, learning_rate, num)
pred = np.dot(x, theta)
print(theta)

J = np.zeros(len(old_theta))
for i, value in enumerate(old_theta):
    J[i] = loss(x, y, value)

fig = plt.figure(figsize=(16, 6))

ax = fig.add_subplot(121)
ax.scatter([x[:,1]], [y], c='b', marker='x')
ax.scatter([x[:,1]], [pred], c='r', marker='o')
ax.set_xlabel('R&D Spend')
ax.set_ylabel('Profit')

ax = fig.add_subplot(122)
ax.plot(iterations, J, 'bo')
ax.set_xlabel('Iterations')
ax.set_ylabel('Loss function')

plt.show()
```


```python
init_theta = np.matrix((0, 0, 0)).T
learning_rate = 5e-6
num = 100000

y = np.matrix([df['Profit'].values]).T
x = np.matrix([np.ones(len(y)), df['R&D Spend'].values, df['Marketing Spend'].values]).T
x = StandardScaler().fit_transform(x)
theta, old_theta, iterations = stochastic_gradient_descent(x, y, init_theta, learning_rate, num)
pred = np.dot(x, theta)
print(theta)

J = np.zeros(len(old_theta))
for i, value in enumerate(old_theta):
    J[i] = loss(x, y, value)

fig = plt.figure(figsize=(16, 6))

ax = fig.add_subplot(121, projection='3d')
ax.scatter(x[:, 1], x[:, 2], y, c='b', s=50, marker='x')
ax.scatter(x[:, 1], x[:, 2], pred, c='r', s=50, marker='o')
ax.set_xlabel('Normalized R&D Spend')
ax.set_ylabel('Normalized Marketing Spend')
ax.set_zlabel('Profit')


ax = fig.add_subplot(122)
ax.plot(iterations, J, 'bo')
ax.set_xlabel('Iterations')
ax.set_ylabel('Loss function')

plt.show()
```

Stochastic gradient descent algorithm can't be used to find a global minimum of the loss function neither.

# Scikit-learn Linear Regression model


```python
y = np.matrix(df['Profit'].values).T
x = np.matrix([np.ones(len(y)), df['R&D Spend'].values]).T
model = LinearRegression(fit_intercept=True)
model.fit(x[:,1], y)
print(f'Intercept: {model.intercept_}')
print(f'Coefficients: {model.coef_}')

xfit = np.linspace(0, 175, 1000)
yfit = model.predict(xfit.reshape(-1, 1))

fig = plt.figure(figsize=(8, 6))

ax = fig.add_subplot(111)
ax.scatter([x[:,1]], [y], c='b', marker='x')
ax.plot(xfit, yfit, 'r')
ax.set_xlabel('R&D Spend')
ax.set_ylabel('Profit')

plt.show()
```


```python
y = np.matrix(df['Profit'].values).T
x = np.matrix([np.ones(len(y)), df['R&D Spend'].values, df['Marketing Spend'].values]).T
x = StandardScaler().fit_transform(x)
model = LinearRegression(fit_intercept=True)
model.fit(x[:,1:], y)
print(f'Intercept: {model.intercept_}')
print(f'Coefficients: {model.coef_}')

xfit_1 = np.linspace(-2, 2, 1000)
xfit_2 = np.linspace(-2, 2, 1000)
xfit = np.column_stack((xfit_1, xfit_2))
yfit = model.predict(xfit)
xfit_1, xfit_2 = np.meshgrid(xfit_1, xfit_2)

fig = plt.figure(figsize=(10, 8))

ax = fig.add_subplot(111, projection='3d')
ax.scatter(x[:,1], x[:,2], y, c='b', marker='x')
ax.plot_surface(xfit_1, xfit_2, yfit, rstride=25, cstride=25, color='red', alpha=0.5)
ax.set_xlabel('Normalized R&D Spend')
ax.set_ylabel('Normalized Marketing Spend')
ax.set_zlabel('Profit')

plt.show()
```

e846b40f253631c0e9efa491085c941ff9c8aabf
ipynb
Jupyter Notebook
Linear Regression/Linear Regression.ipynb
mickuz/machine-learning
369817515e9d896c3fd6a13473daf9dafef6e02f
MIT
Linear Regression/Linear Regression.ipynb
mickuz/machine-learning
369817515e9d896c3fd6a13473daf9dafef6e02f
MIT
Linear Regression/Linear Regression.ipynb
mickuz/machine-learning
369817515e9d896c3fd6a13473daf9dafef6e02f
MIT
Qwen/Qwen-72B
1. YES
2. YES