# Homework 2
## BIOST 558 Spring 2020 
### Juan Solorio


```python
# needed libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn import linear_model
import statsmodels.api as sm

%matplotlib inline
plt.style.use('ggplot')
```

### Exercise 1
#### Gradient Descent - Theory (part 1 answers)


We have the objective equation

\begin{equation}
F(\beta) = \frac{1}n\sum_{i=1}^n(y_i - x^T_i\beta)^2 + \lambda\|\beta\|^2_2
\end{equation}

or expanded:

\begin{equation}
F(\beta) = \frac{1}n\sum_{i=1}^n(y_i - \sum_{j=1}^dx_{ij}\beta_j)^2 + \lambda\sum_{j=1}^d\beta_j^2
\end{equation}

which we want to minimize with respect to $\beta$, meaning we want to take the derivative of $F(\beta)$ and set it equal to zero. 

The derivative for n=1 and d=1 yields:
\begin{equation}
\frac{dF}{d\beta} = \frac{2}n(y - x\beta) + 2\lambda\beta
\end{equation}

In Matrix terms, $F(\beta)$ can be interpreted as:
\begin{equation}
F(\beta) = (Y - X\beta)^T(Y-X\beta) + \lambda\beta^T\beta
\end{equation}
\begin{equation}
= y^Ty - \beta^TX^Ty-y^TX\beta+\beta^TX^TX\beta+\lambda\beta^T\beta
\end{equation}
\begin{equation}
= y^Ty - \beta^TX^Ty-y^TX\beta+\beta^TX^TX\beta+\beta^T\lambda I\beta
\end{equation}
\begin{equation}
= y^Ty - 2\beta^TX^Ty + \beta^T(X^TX + \lambda I)\beta
\end{equation}


Taking the derivative of the matrix form of $F(\beta)$ leads us to the following
\end{equation}
\begin{equation}
\frac{d}{d\beta}= - 2X^Ty + 2(X^TX + \lambda I)\beta
\end{equation}




#### Gradient Descent - Algorithm Code


```python
# Load the data
hitters = pd.read_csv('https://raw.githubusercontent.com/selva86/datasets/master/Hitters.csv',
                      sep=',', header=0)
hitters = hitters.dropna()
```


```python
# Creating matrix X with the predictors and 
# and vector y with the response 
X = hitters.drop('Salary', axis=1)
X = pd.get_dummies(X, drop_first=True)
y = hitters.Salary
```


```python
# Dividing the data into train and test sets.
# By default,  it is a 75-25 split between train-test
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)

# Standarizing the data
X_train = np.array((X_train-X_train.mean())/(X_train.max()-X_train.min()))
y_train = np.array((y_train-y_train.mean())/(y_train.max()-y_train.min()))
```


```python
def betas(X,y,lamda=0.05):
    '''
    Function to calculate betas from the matrix solution
    '''
    I = np.identity(len(X[0]))
    return np.dot(np.linalg.inv(np.dot(X.T,X) + lamda*I),np.dot(X.T,y))
```


```python
def Fx_ridge(X,y,beta, lamda):
    '''
    # RSS = (y-XB).T(y-XB)+lamda*B.T*B
    This is the objective function for Ridge regression
    '''
    n = len(y)
    lin = y - (X @ beta)
    reg = lamda*(beta.T @ beta)
    return (1/n *(lin.T @ lin) + reg)
```


```python
def computegrad(X, y, beta,lamda):
    '''
    Function for the gradient of the ridge objective function
    '''
    n = len(X[1])
    I = np.identity(len(X_train[1]))
    lin = (X_train.T @ y_train)/n
    reg = ((X_train.T @ X_train)/n + lamda*I) @ beta
    return 2*(reg-lin)
```


```python
def graddescent(X,y,t,beta,lamda,max_iter=1000):
    '''
    Gradient descent function that takes in X,y,t,beta,lamda,max_iter
    returns list of betas, grandient, objective fx values
    '''
    grad = computegrad(X,y,beta,lamda)
    fx = Fx_ridge(X,y,beta,lamda)
    fx_vals = [fx]
    beta_vals = [beta]
    grad_vals = [np.linalg.norm(grad)]
    for i in range(max_iter):
        beta = beta - t*grad
        grad = computegrad(X,y,beta,lamda)
        fx = Fx_ridge(X,y,beta,lamda)
        beta_vals.append(beta)
        grad_vals.append(np.linalg.norm(grad))
        fx_vals.append(fx)
    return [beta_vals,grad_vals,fx_vals]
    
```


```python
def graddescent_eps(X,y,t,beta,lamda,eps,max_iter=1000):
    '''
    Gradient descent function that takes in X,y,t,beta,lamda,max_iter
    returns list of betas, grandient, objective fx values
    '''
    grad = computegrad(X,y,beta,lamda)
    fx = Fx_ridge(X,y,beta,lamda)
    fx_vals = [fx]
    beta_vals = [beta]
    grad_vals = [grad]
    iter_val = 0
    while np.linalg.norm(grad) > eps and iter_val < max_iter:
        beta = beta - t*grad
        grad = computegrad(X,y,beta,lamda)
        fx = Fx_ridge(X,y,beta,lamda)
        beta_vals.append(beta)
        grad_vals.append(grad)
        fx_vals.append(fx)
    return [beta_vals,grad_vals,fx_vals]
    
```

##### Checking the Gradient Descent and Objective Ridge Fx for $\lambda = -5.00$


```python
n=len(X_train[1])
beta = np.zeros(n)
t=0.05
lamda=-5

grad_desc_n5 = graddescent(X_train, y_train,t,beta,lamda)
ts = np.array(range(len(grad_desc_n5[1])))
plt.scatter(x=ts,y=np.array(grad_desc_n5[2]))
plt.show()
```

From the figure we can observe that with $\lambda < 0$, the Ridge objective function diverges to $-\infty$, which was to be expected given that it would cause the gradient descent to become more of a gradient 'ascent'.

##### Checking the Gradient Descent and Objective Ridge Fx for $\lambda = 0.05$


```python
n=len(X_train[1])
beta = np.zeros(n)
t=0.05
lamda=0.05

grad_desc_05 = graddescent(X_train, y_train,t,beta,lamda)
```


```python
df_fx = pd.DataFrame(grad_desc_05[2])
df_grad = pd.DataFrame(grad_desc_05[1])
df_beta = pd.DataFrame(grad_desc_05[0])
df_fx.plot(legend=False,figsize=(8,6),title='Ridge Objective Fx')
plt.xlabel("t")
plt.show()
```


```python
df_beta.plot(subplots=True,layout=(4,5),legend=False,figsize=(14,10), title=r"$\beta$ Individial Predictors' Gradient Descent")
plt.subplots_adjust(hspace=0.35)
plt.show()
```

As can be observed in the $"Ridge\ Objective\ Fx"$ chart and the $"\beta\ Individial\ Predictors'\ Gradient\ Descent"$ charts, for the $\lambda = 0.05$, we can see each of these values converging, in the case of the $\beta s$ converge to some number and for the $F(\beta)$ to about 0.032.

#### Comparing sklearn.linear_model.Ridge $\beta$ to those values from the defined functions


```python
# sklearn Ridge regression betas
ridge = linear_model.Ridge(alpha=0.05)
model = ridge.fit(X_train,y_train)
model.coef_
```




    array([-0.49423255,  0.61521257,  0.14279993, -0.12541456,  0.0084655 ,
            0.31010389, -0.08027074, -0.36881245,  0.56366317, -0.05996115,
            0.52130315,  0.05585199, -0.4589716 ,  0.14773404,  0.06993996,
           -0.08699545,  0.01999316, -0.0504822 , -0.00686063])




```python
# betas calculated through my betas function
B = betas(X_train,y_train)
B
```




    array([-0.49423255,  0.61521257,  0.14279993, -0.12541456,  0.0084655 ,
            0.31010389, -0.08027074, -0.36881245,  0.56366317, -0.05996115,
            0.52130315,  0.05585199, -0.4589716 ,  0.14773404,  0.06993996,
           -0.08699545,  0.01999316, -0.0504822 , -0.00686063])




```python
print("sklearn: %f \t Vectorize Ridge: %f \t Grad Desc: %f"%(
    Fx_ridge(X_train,y_train,model.coef_,lamda),Fx_ridge(X_train,y_train,B,lamda),np.array(grad_desc_05[2])[-1]))
```

    sklearn: 0.107533 	 Vectorize Ridge: 0.107533 	 Grad Desc: 0.031999


The values calculated by the vectorize Ridge function are quite close to those from the sklearn Ridge function.
However the values calculated by my gradient descent were lower than these two previous values.

#### Adding a stopping criteria $\|\nabla F(\beta)\| <$ $\epsilon = 0.005$ 


```python
n=len(X_train[1])
beta = np.zeros(n)
t=0.2
lamda=0.05
eps = 0.005
grad_desc_eps = graddescent_eps(X_train, y_train,t,beta,lamda,eps)

df_fx = pd.DataFrame(grad_desc_eps[2])
df_grad = pd.DataFrame(grad_desc_eps[1])
df_fx.plot(legend=False,figsize=(8,6),title='Ridge Objective Fx')
plt.xlabel("t")
plt.show()
```

The new criteria allows for the gradient descent process to stop sooner as it converges without the need to continue iterating.

### Exercise 2


```python
# (a) Read in the dataset. The data can be downloaded from this url: http://www-bcf.usc.
# edu/~gareth/ISL/Auto.csv When reading in the data use the option na values=’?’.
# Then drop all NaN values using dropna().

# Load the data
Auto = pd.read_csv("http://faculty.marshall.usc.edu/gareth-james/ISL/Auto.csv",
                      sep=',', header=0, na_values='?')
Auto = Auto.dropna()
Auto['constant']=1
```


```python
# (b) Use the OLS function from the statsmodels package to perform a simple linear regression with mpg as the response and weight as the predictor. Be sure to include an
# intercept. Use the summary() attribute to print the results.

X = Auto.loc[:,['constant','weight']]
Y = Auto.mpg
lm_mpg_wgt = sm.OLS(Y,X)

lm_mpg_wgt_model = lm_mpg_wgt.fit()
print(lm_mpg_wgt_model.summary())
```

                                OLS Regression Results                            
    ==============================================================================
    Dep. Variable:                    mpg   R-squared:                       0.693
    Model:                            OLS   Adj. R-squared:                  0.692
    Method:                 Least Squares   F-statistic:                     878.8
    Date:                Fri, 17 Apr 2020   Prob (F-statistic):          6.02e-102
    Time:                        20:56:37   Log-Likelihood:                -1130.0
    No. Observations:                 392   AIC:                             2264.
    Df Residuals:                     390   BIC:                             2272.
    Df Model:                           1                                         
    Covariance Type:            nonrobust                                         
    ==============================================================================
                     coef    std err          t      P>|t|      [0.025      0.975]
    ------------------------------------------------------------------------------
    constant      46.2165      0.799     57.867      0.000      44.646      47.787
    weight        -0.0076      0.000    -29.645      0.000      -0.008      -0.007
    ==============================================================================
    Omnibus:                       41.682   Durbin-Watson:                   0.808
    Prob(Omnibus):                  0.000   Jarque-Bera (JB):               60.039
    Skew:                           0.727   Prob(JB):                     9.18e-14
    Kurtosis:                       4.251   Cond. No.                     1.13e+04
    ==============================================================================
    
    Warnings:
    [1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
    [2] The condition number is large, 1.13e+04. This might indicate that there are
    strong multicollinearity or other numerical problems.


- i) From the summary statistics we can observe there is a strong relation between Weight and mpg. 

- ii) From the summary we see that the p-value suggest a very strong statistical significance for the linear relation between MPG and Weight. With a correlation of -0.83

- iii) The relationship between MPG and Weight is negative.


```python
fig, ax = plt.subplots()
fig = sm.graphics.plot_fit(lm_mpg_wgt_model, 1, ax=ax)
ax.set_xlabel("Weight")
ax.set_ylabel("mpg")
ax.set_title("Linear Regression")
plt.show()
```


```python
# (d) Plot the residuals vs. fitted values. Comment on any problems you see with the fit.
fig, ax = plt.subplots()
lm_mpg_wgt_fitted = lm_mpg_wgt_model.fittedvalues
lm_mpg_wgt_res = lm_mpg_wgt_model.resid
fig = sns.residplot(lm_mpg_wgt_fitted,Auto.columns[0],data=Auto,
             lowess=True,
             scatter_kws={'alpha':0.5},
             line_kws={'color': 'red', 'lw': 1, 'alpha': 0.8})
ax.set_xlabel("Fitted")
ax.set_ylabel("Residuals")
ax.set_title("Residuals vs Fitted")
plt.show()
```

The red lines across the data shows a "bow-shape" or curve which indicates we are not capturing some of the non-linear aspects of the model.

### Exercise 3


```python
# (a) Produce a scatterplot matrix which includes all of the variables in the data set using
# pandas.plotting.scatter matrix
pd.plotting.scatter_matrix(Auto, alpha=0.5, figsize=(14,14))
plt.show()
```


```python
# (b) Compute the matrix of correlations between the variables using the corr() attribute
# in Pandas.

corrMatrix = Auto.corr()
sns.heatmap(corrMatrix,annot=True)
plt.show()
```


```python
# (c) Use the OLS function from the statsmodels package to perform a multiple linear
# regression with mpg as the response and all other variables except name as the predictors.
# Be sure to include an intercept.
Xmlt = Auto.loc[:,['constant','cylinders','displacement','horsepower','weight','acceleration','year','origin']]
lm_mpg_mlt = sm.OLS(Y,Xmlt)

lm_mpg_mlt_model = lm_mpg_mlt.fit()
print(lm_mpg_mlt_model.summary())
```

                                OLS Regression Results                            
    ==============================================================================
    Dep. Variable:                    mpg   R-squared:                       0.821
    Model:                            OLS   Adj. R-squared:                  0.818
    Method:                 Least Squares   F-statistic:                     252.4
    Date:                Fri, 17 Apr 2020   Prob (F-statistic):          2.04e-139
    Time:                        20:57:02   Log-Likelihood:                -1023.5
    No. Observations:                 392   AIC:                             2063.
    Df Residuals:                     384   BIC:                             2095.
    Df Model:                           7                                         
    Covariance Type:            nonrobust                                         
    ================================================================================
                       coef    std err          t      P>|t|      [0.025      0.975]
    --------------------------------------------------------------------------------
    constant       -17.2184      4.644     -3.707      0.000     -26.350      -8.087
    cylinders       -0.4934      0.323     -1.526      0.128      -1.129       0.142
    displacement     0.0199      0.008      2.647      0.008       0.005       0.035
    horsepower      -0.0170      0.014     -1.230      0.220      -0.044       0.010
    weight          -0.0065      0.001     -9.929      0.000      -0.008      -0.005
    acceleration     0.0806      0.099      0.815      0.415      -0.114       0.275
    year             0.7508      0.051     14.729      0.000       0.651       0.851
    origin           1.4261      0.278      5.127      0.000       0.879       1.973
    ==============================================================================
    Omnibus:                       31.906   Durbin-Watson:                   1.309
    Prob(Omnibus):                  0.000   Jarque-Bera (JB):               53.100
    Skew:                           0.529   Prob(JB):                     2.95e-12
    Kurtosis:                       4.460   Cond. No.                     8.59e+04
    ==============================================================================
    
    Warnings:
    [1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
    [2] The condition number is large, 8.59e+04. This might indicate that there are
    strong multicollinearity or other numerical problems.



```python
# (d) Plot the residuals vs. fitted values. Comment on any problems you see with the fit.
fig, ax = plt.subplots()
lm_mpg_mlt_fitted = lm_mpg_mlt_model.fittedvalues
lm_mpg_mlt_res = lm_mpg_mlt_model.resid
fig = sns.residplot(lm_mpg_mlt_fitted,Auto.columns[0],data=Auto,
             lowess=True,
             scatter_kws={'alpha':0.5},
             line_kws={'color': 'red', 'lw': 1, 'alpha': 0.8})
ax.set_xlabel("Fitted")
ax.set_ylabel("Residuals")
ax.set_title("Residuals vs Fitted")
plt.show()

```

Even from the multivariate model, the Residuals vs Fitted plot is still showing some aspects of the model that are not accounted for. This might mean we could be overlooking some sort of transformation to possibly allow for a better fit.


```python
# (e) Statsmodels allows you to fit models using R-style formulas. See http://www.statsmodels.
# org/dev/example_formulas.html. Use the * and : symbols to fit linear regression
# models with interaction effects. Do any interactions appear to be statistically significant?

import statsmodels.formula.api as smf
Auto_x = Auto.drop(['name','constant'],axis=1)
lm_mpg_mult = smf.ols(formula = 'mpg ~ cylinders * displacement * horsepower * weight * acceleration * year * origin', data=Auto_x)
res_mult = lm_mpg_mult.fit()
print(res_mult.summary())
```

                                OLS Regression Results                            
    ==============================================================================
    Dep. Variable:                    mpg   R-squared:                       0.931
    Model:                            OLS   Adj. R-squared:                  0.906
    Method:                 Least Squares   F-statistic:                     38.51
    Date:                Fri, 17 Apr 2020   Prob (F-statistic):          1.49e-123
    Time:                        20:57:10   Log-Likelihood:                -838.23
    No. Observations:                 392   AIC:                             1880.
    Df Residuals:                     290   BIC:                             2286.
    Df Model:                         101                                         
    Covariance Type:            nonrobust                                         
    =====================================================================================================================================
                                                                            coef    std err          t      P>|t|      [0.025      0.975]
    -------------------------------------------------------------------------------------------------------------------------------------
    Intercept                                                            -0.0172      0.395     -0.043      0.965      -0.795       0.761
    cylinders                                                            -0.9849      3.107     -0.317      0.751      -7.100       5.130
    displacement                                                         15.6423     26.104      0.599      0.549     -35.735      67.019
    cylinders:displacement                                                9.9087      7.405      1.338      0.182      -4.666      24.483
    horsepower                                                            3.4947     23.826      0.147      0.883     -43.399      50.388
    cylinders:horsepower                                                  0.3706      8.546      0.043      0.965     -16.450      17.191
    displacement:horsepower                                              -0.2288      0.317     -0.722      0.471      -0.852       0.395
    cylinders:displacement:horsepower                                    -0.1061      0.094     -1.128      0.260      -0.291       0.079
    weight                                                               -0.5506      0.852     -0.647      0.518      -2.227       1.125
    cylinders:weight                                                     -0.3717      0.284     -1.309      0.192      -0.931       0.187
    displacement:weight                                                  -0.0100      0.022     -0.455      0.649      -0.053       0.033
    cylinders:displacement:weight                                         0.0016      0.004      0.389      0.698      -0.007       0.010
    horsepower:weight                                                     0.0024      0.011      0.224      0.823      -0.019       0.024
    cylinders:horsepower:weight                                           0.0040      0.004      1.084      0.279      -0.003       0.011
    displacement:horsepower:weight                                    -1.781e-05      0.000     -0.081      0.935      -0.000       0.000
    cylinders:displacement:horsepower:weight                           2.426e-05   4.09e-05      0.593      0.553   -5.62e-05       0.000
    acceleration                                                        -24.8380     46.061     -0.539      0.590    -115.494      65.818
    cylinders:acceleration                                                2.7590     24.784      0.111      0.911     -46.020      51.538
    displacement:acceleration                                            -1.0383      1.458     -0.712      0.477      -3.908       1.832
    cylinders:displacement:acceleration                                  -0.7078      0.511     -1.384      0.167      -1.714       0.299
    horsepower:acceleration                                               0.0149      1.697      0.009      0.993      -3.326       3.355
    cylinders:horsepower:acceleration                                    -0.1210      0.636     -0.190      0.849      -1.373       1.131
    displacement:horsepower:acceleration                                  0.0271      0.024      1.153      0.250      -0.019       0.073
    cylinders:displacement:horsepower:acceleration                        0.0049      0.006      0.767      0.444      -0.008       0.018
    weight:acceleration                                                   0.0331      0.062      0.534      0.594      -0.089       0.155
    cylinders:weight:acceleration                                         0.0279      0.018      1.549      0.123      -0.008       0.063
    displacement:weight:acceleration                                     -0.0004      0.001     -0.275      0.784      -0.003       0.002
    cylinders:displacement:weight:acceleration                            0.0002      0.000      0.759      0.449      -0.000       0.001
    horsepower:weight:acceleration                                     7.089e-05      0.001      0.083      0.934      -0.002       0.002
    cylinders:horsepower:weight:acceleration                             -0.0003      0.000     -1.293      0.197      -0.001       0.000
    displacement:horsepower:weight:acceleration                        5.365e-06   1.68e-05      0.319      0.750   -2.77e-05    3.85e-05
    cylinders:displacement:horsepower:weight:acceleration             -2.933e-06   3.04e-06     -0.965      0.335   -8.91e-06    3.05e-06
    year                                                                 -7.7709      5.452     -1.425      0.155     -18.502       2.960
    cylinders:year                                                       -2.2045      1.821     -1.210      0.227      -5.789       1.380
    displacement:year                                                    -0.1631      0.344     -0.473      0.636      -0.841       0.515
    cylinders:displacement:year                                          -0.1110      0.097     -1.140      0.255      -0.303       0.081
    horsepower:year                                                       0.0346      0.322      0.107      0.915      -0.600       0.669
    cylinders:horsepower:year                                             0.0331      0.121      0.275      0.784      -0.204       0.270
    displacement:horsepower:year                                          0.0018      0.004      0.411      0.681      -0.007       0.010
    cylinders:displacement:horsepower:year                                0.0013      0.001      1.038      0.300      -0.001       0.004
    weight:year                                                           0.0100      0.015      0.674      0.501      -0.019       0.039
    cylinders:weight:year                                                 0.0061      0.004      1.670      0.096      -0.001       0.013
    displacement:weight:year                                           7.838e-05      0.000      0.297      0.767      -0.000       0.001
    cylinders:displacement:weight:year                                -2.061e-05   4.85e-05     -0.425      0.671      -0.000    7.49e-05
    horsepower:weight:year                                            -2.278e-05      0.000     -0.142      0.887      -0.000       0.000
    cylinders:horsepower:weight:year                                   -8.25e-05   4.81e-05     -1.715      0.087      -0.000    1.22e-05
    displacement:horsepower:weight:year                                7.245e-07   2.65e-06      0.274      0.784   -4.48e-06    5.93e-06
    cylinders:displacement:horsepower:weight:year                     -2.609e-07   4.95e-07     -0.527      0.598   -1.23e-06    7.13e-07
    acceleration:year                                                     0.8048      0.649      1.240      0.216      -0.472       2.082
    cylinders:acceleration:year                                           0.1015      0.330      0.308      0.759      -0.547       0.750
    displacement:acceleration:year                                        0.0271      0.021      1.284      0.200      -0.014       0.069
    cylinders:displacement:acceleration:year                              0.0042      0.007      0.628      0.531      -0.009       0.017
    horsepower:acceleration:year                                         -0.0057      0.024     -0.236      0.813      -0.053       0.041
    cylinders:horsepower:acceleration:year                               -0.0005      0.009     -0.055      0.956      -0.018       0.017
    displacement:horsepower:acceleration:year                            -0.0004      0.000     -1.253      0.211      -0.001       0.000
    cylinders:displacement:horsepower:acceleration:year               -2.828e-05   8.87e-05     -0.319      0.750      -0.000       0.000
    weight:acceleration:year                                             -0.0008      0.001     -0.799      0.425      -0.003       0.001
    cylinders:weight:acceleration:year                                   -0.0004      0.000     -1.703      0.090      -0.001    6.22e-05
    displacement:weight:acceleration:year                              3.713e-06    1.6e-05      0.232      0.817   -2.78e-05    3.53e-05
    cylinders:displacement:weight:acceleration:year                   -1.157e-06   2.54e-06     -0.456      0.649   -6.15e-06    3.84e-06
    horsepower:weight:acceleration:year                                9.533e-08   1.19e-05      0.008      0.994   -2.32e-05    2.34e-05
    cylinders:horsepower:weight:acceleration:year                      5.662e-06   3.23e-06      1.751      0.081   -7.03e-07     1.2e-05
    displacement:horsepower:weight:acceleration:year                  -6.705e-08   2.04e-07     -0.328      0.743   -4.69e-07    3.35e-07
    cylinders:displacement:horsepower:weight:acceleration:year         2.691e-08   3.69e-08      0.728      0.467   -4.58e-08    9.96e-08
    origin                                                               -1.6028      3.091     -0.518      0.605      -7.687       4.481
    cylinders:origin                                                     -0.7671      2.486     -0.309      0.758      -5.660       4.126
    displacement:origin                                                  10.2550     21.860      0.469      0.639     -32.769      53.279
    cylinders:displacement:origin                                       -11.6548     12.294     -0.948      0.344     -35.851      12.542
    horsepower:origin                                                     2.9976     21.387      0.140      0.889     -39.095      45.090
    cylinders:horsepower:origin                                          -1.6308     11.010     -0.148      0.882     -23.300      20.039
    displacement:horsepower:origin                                       -0.1666      0.266     -0.626      0.532      -0.690       0.357
    cylinders:displacement:horsepower:origin                              0.1478      0.152      0.975      0.330      -0.151       0.446
    weight:origin                                                        -0.3470      0.728     -0.477      0.634      -1.779       1.085
    cylinders:weight:origin                                               0.4401      0.414      1.063      0.289      -0.375       1.255
    displacement:weight:origin                                            0.0092      0.020      0.461      0.645      -0.030       0.048
    cylinders:displacement:weight:origin                                 -0.0017      0.005     -0.366      0.714      -0.011       0.008
    horsepower:weight:origin                                              0.0054      0.013      0.428      0.669      -0.019       0.030
    cylinders:horsepower:weight:origin                                   -0.0046      0.006     -0.830      0.407      -0.015       0.006
    displacement:horsepower:weight:origin                              6.378e-05      0.000      0.321      0.748      -0.000       0.000
    cylinders:displacement:horsepower:weight:origin                   -2.918e-05    4.8e-05     -0.608      0.544      -0.000    6.53e-05
    acceleration:origin                                                 -23.8165     44.972     -0.530      0.597    -112.329      64.696
    cylinders:acceleration:origin                                         6.8451     20.638      0.332      0.740     -33.774      47.464
    displacement:acceleration:origin                                     -0.6671      1.193     -0.559      0.576      -3.015       1.680
    cylinders:displacement:acceleration:origin                            0.7792      0.746      1.044      0.297      -0.690       2.248
    horsepower:acceleration:origin                                        0.0620      1.564      0.040      0.968      -3.016       3.140
    cylinders:horsepower:acceleration:origin                              0.0674      0.738      0.091      0.927      -1.384       1.519
    displacement:horsepower:acceleration:origin                           0.0011      0.017      0.065      0.948      -0.033       0.035
    cylinders:displacement:horsepower:acceleration:origin                -0.0075      0.009     -0.818      0.414      -0.026       0.011
    weight:acceleration:origin                                            0.0212      0.055      0.385      0.701      -0.087       0.130
    cylinders:weight:acceleration:origin                                 -0.0292      0.028     -1.032      0.303      -0.085       0.026
    displacement:weight:acceleration:origin                               0.0005      0.001      0.395      0.693      -0.002       0.003
    cylinders:displacement:weight:acceleration:origin                    -0.0002      0.000     -0.661      0.509      -0.001       0.000
    horsepower:weight:acceleration:origin                                -0.0006      0.001     -0.636      0.525      -0.002       0.001
    cylinders:horsepower:weight:acceleration:origin                       0.0004      0.000      0.929      0.354      -0.000       0.001
    displacement:horsepower:weight:acceleration:origin                -9.374e-06   1.57e-05     -0.596      0.552   -4.03e-05    2.16e-05
    cylinders:displacement:horsepower:weight:acceleration:origin       3.341e-06   3.28e-06      1.019      0.309   -3.11e-06    9.79e-06
    year:origin                                                          -6.1203      4.912     -1.246      0.214     -15.789       3.548
    cylinders:year:origin                                                 4.3981      2.511      1.752      0.081      -0.544       9.340
    displacement:year:origin                                             -0.1078      0.289     -0.373      0.709      -0.677       0.461
    cylinders:displacement:year:origin                                    0.1233      0.162      0.761      0.447      -0.196       0.442
    horsepower:year:origin                                                0.0188      0.289      0.065      0.948      -0.551       0.588
    cylinders:horsepower:year:origin                                     -0.0343      0.153     -0.224      0.823      -0.335       0.267
    displacement:horsepower:year:origin                                   0.0028      0.004      0.792      0.429      -0.004       0.010
    cylinders:displacement:horsepower:year:origin                        -0.0018      0.002     -0.879      0.380      -0.006       0.002
    weight:year:origin                                                    0.0076      0.011      0.699      0.485      -0.014       0.029
    cylinders:weight:year:origin                                         -0.0079      0.005     -1.559      0.120      -0.018       0.002
    displacement:weight:year:origin                                   -9.245e-05      0.000     -0.397      0.692      -0.001       0.000
    cylinders:displacement:weight:year:origin                          2.569e-05   5.68e-05      0.452      0.651   -8.61e-05       0.000
    horsepower:weight:year:origin                                        -0.0001      0.000     -0.995      0.321      -0.000       0.000
    cylinders:horsepower:weight:year:origin                            9.742e-05   6.83e-05      1.427      0.155   -3.69e-05       0.000
    displacement:horsepower:weight:year:origin                        -1.063e-06   2.32e-06     -0.457      0.648   -5.64e-06    3.51e-06
    cylinders:displacement:horsepower:weight:year:origin               2.911e-07   5.94e-07      0.490      0.625   -8.78e-07    1.46e-06
    acceleration:year:origin                                              0.6899      0.627      1.101      0.272      -0.544       1.923
    cylinders:acceleration:year:origin                                   -0.3570      0.283     -1.263      0.208      -0.914       0.200
    displacement:acceleration:year:origin                                -0.0086      0.018     -0.489      0.625      -0.043       0.026
    cylinders:displacement:acceleration:year:origin                      -0.0045      0.010     -0.457      0.648      -0.024       0.015
    horsepower:acceleration:year:origin                                  -0.0030      0.020     -0.151      0.880      -0.043       0.037
    cylinders:horsepower:acceleration:year:origin                         0.0022      0.010      0.218      0.827      -0.018       0.022
    displacement:horsepower:acceleration:year:origin                   6.517e-05      0.000      0.272      0.786      -0.000       0.001
    cylinders:displacement:horsepower:acceleration:year:origin         5.796e-05      0.000      0.464      0.643      -0.000       0.000
    weight:acceleration:year:origin                                      -0.0003      0.001     -0.400      0.689      -0.002       0.001
    cylinders:weight:acceleration:year:origin                             0.0005      0.000      1.353      0.177      -0.000       0.001
    displacement:weight:acceleration:year:origin                      -3.585e-06    1.4e-05     -0.255      0.799   -3.12e-05     2.4e-05
    cylinders:displacement:weight:acceleration:year:origin             9.195e-07      3e-06      0.306      0.760   -4.99e-06    6.83e-06
    horsepower:weight:acceleration:year:origin                         1.063e-05   1.09e-05      0.978      0.329   -1.08e-05     3.2e-05
    cylinders:horsepower:weight:acceleration:year:origin              -6.546e-06   4.75e-06     -1.377      0.170   -1.59e-05    2.81e-06
    displacement:horsepower:weight:acceleration:year:origin            1.052e-07   1.88e-07      0.560      0.576   -2.65e-07    4.75e-07
    cylinders:displacement:horsepower:weight:acceleration:year:origin -3.037e-08   4.04e-08     -0.751      0.453    -1.1e-07    4.92e-08
    ==============================================================================
    Omnibus:                       45.623   Durbin-Watson:                   1.752
    Prob(Omnibus):                  0.000   Jarque-Bera (JB):              129.382
    Skew:                           0.532   Prob(JB):                     8.04e-29
    Kurtosis:                       5.605   Cond. No.                     3.52e+16
    ==============================================================================
    
    Warnings:
    [1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
    [2] The condition number is large, 3.52e+16. This might indicate that there are
    strong multicollinearity or other numerical problems.



```python
# (f) Try a few different transformations of the variables, such as log(X), 1/X,
# sqrt(X), X2. Comment on your findings.
Auto_log = np.log(Auto_x.drop(['year','origin'],axis=1))
Auto_inv = 1/Auto_x.drop(['year','origin'],axis=1)
Auto_sqrt= np.sqrt(Auto_x.drop(['year','origin'],axis=1))

```


```python
lm_log = smf.ols(formula = 'mpg ~ cylinders * displacement * horsepower * weight * acceleration', data=Auto_log)
lm_log_model = lm_log.fit()
print(lm_log_model.summary())

fig, ax = plt.subplots()
lm_log_fitted = lm_log_model.fittedvalues
lm_log_res = lm_log_model.resid
fig = sns.residplot(lm_log_fitted,Auto_log.columns[0], data=Auto_log,
             lowess=True,
             scatter_kws={'alpha':0.5},
             line_kws={'color': 'red', 'lw': 1, 'alpha': 0.8})
ax.set_xlabel("Fitted")
ax.set_ylabel("Residuals")
ax.set_title("Residuals vs Fitted")
plt.show()
```

By conducting a Log transformation on the data, we can see that the residuals now present a much better fit for the Fitted values. We even see from the $p-values$ that each predictor and the interactions show statistical significance in their relation to mpg.

fc3dedf13c500e8227c451e7432bc11a5fb7f2b9
ipynb
Jupyter Notebook
1-Python/.ipynb_checkpoints/Homework #2-checkpoint.ipynb
JUAN-SOLORIO/UW-MachineLearning
60cf1474bce45dd541d3fb60eb3b2a2eeaa9ca3c
RSA-MD
1-Python/.ipynb_checkpoints/Homework #2-checkpoint.ipynb
JUAN-SOLORIO/UW-MachineLearning
60cf1474bce45dd541d3fb60eb3b2a2eeaa9ca3c
RSA-MD
1-Python/.ipynb_checkpoints/Homework #2-checkpoint.ipynb
JUAN-SOLORIO/UW-MachineLearning
60cf1474bce45dd541d3fb60eb3b2a2eeaa9ca3c
RSA-MD
Qwen/Qwen-72B
1. YES
2. YES