<filename>A_HANDS-ON_GUIDE_TO_REGRESSION_WITH_FASTAI.py
# -*- coding: utf-8 -*-
"""Published-Regression_Using_Fastai.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/17VOQ78Wwj2ByP98M6ojhqpkq0usp3Udt

#Regression With Fastai

---

Published at Analytics India Magazine -- [A HANDS-ON GUIDE TO REGRESSION WITH FAST.AI](https://analyticsindiamag.com/a-hands-on-guide-to-regression-with-fast-ai/)

**Getting Started With Regression**

Regression With Fast.ai in 7 simple steps:

* Importing the libraries
* Creating a TabularList
* Initialising Neural Network
* Training the model
* Evaluating the model
* A simple analysis on the predictions of the validation set
* Predicting using the network

##Importing All Major Libraries
"""

import pandas as pd
import numpy as np
from fastai.tabular import *

"""**The fastai.tabular package includes all the modules that are necessary for processing tabular data.**

## Importing The Data
"""

#Reading the datasets from excel sheet
training_set = pd.read_excel("Data_Train.xlsx")
test_set = pd.read_excel("Data_Test.xlsx")

"""## Understanding The Data"""

training_set.head(5)

# Checking the number of rows
print("\n\nNumber of observations in the datasets :\n",'#' * 40)

print("\nTraining Set : ",len(training_set))
print("Test Set : ",len(test_set))

# checking the number of features in the Datasets
print("\n\nNumber of features in the datasets :\n",'#' * 40)

print("\nTraining Set : ",len(training_set.columns))
print("Test Set : ",len(test_set.columns))

# checking the features in the Datasets
print("\n\nFeatures in the datasets :\n",'#' * 40)

print("\nTraining Set : ", list(training_set.columns))
print("Test Set : ",list(test_set.columns))

# Checking the data types of features
print("\n\nDatatypes of features in the datasets :\n",'#' * 40)

print("\nTraining Set : ", training_set.dtypes)
print("\nTest Set : ",test_set.dtypes)

# checking for NaNs or empty cells
print("\n\nEmpty cells or Nans in the datasets :\n",'#' * 40)

print("\nTraining Set : ",training_set.isnull().values.any())
print("\nTest Set : ",test_set.isnull().values.any())

# checking for NaNs or empty cells by column
print("\n\nNumber of empty cells or Nans in the datasets :\n",'#' * 40)

print("\nTraining Set : ","\n", training_set.isnull().sum())
print("\nTest Set : ",test_set.isnull().sum())

#Displaying dataset information
print("\n\nInfo:\n",'#' * 40)

training_set.info()

"""### Exploring Categorical features"""

# Non categorical Features in The dataset
training_set.select_dtypes(['int','float']).columns

#Categotical Features in The Dataset
training_set.select_dtypes('object').columns

#The Unique values in each of the categorical features 

all_brands = list(training_set.Name) + list(test_set.Name)

all_locations = list(training_set.Location) + list(test_set.Location)

all_fuel_types = list(training_set.Fuel_Type) + list(test_set.Fuel_Type)

all_transmissions = list(training_set.Transmission) + list(test_set.Transmission)

all_owner_types = list(training_set.Owner_Type) + list(test_set.Owner_Type)



print("\nNumber Of Unique Values In Name : \n ", len(set(all_brands)))
#print("\nThe Unique Values In Name : \n ", set(all_brands))

print("\nNumber Of Unique Values In Location : \n ", len(set(all_locations)))
print("\nThe Unique Values In Location : \n ", set(all_locations) )

print("\nNumber Of Unique Values In Fuel_Type : \n ", len(set(all_fuel_types)))
print("\nThe Unique Values In Fuel_Type : \n ", set(all_fuel_types) )

print("\nNumber Of Unique Values In Transmission : \n ", len(set(all_transmissions)))
print("\nThe Unique Values In Transmission : \n ", set(all_transmissions) )

print("\nNumber Of Unique Values In Owner_Type : \n ", len(set(all_owner_types)))
print("\nThe Unique Values In Owner_Type : \n ", set(all_owner_types) )

"""## Feature Generation And Dataset Restructuring"""

#Based on the information gathered from the data, lets simplify and restructure it.

def restructure(data):
  
  names = list(data.Name)
  
  brand = []
  model = []
  
  #Splitting The Column 'Name'
  for i in range(len(names)):
    try:
      brand.append(names[i].split(" ")[0])
      try:
        model.append(" ".join(names[i].split(" ")[1:]).strip())
      except:
        pass
    except:
        print("ERR ! - ", names[i], "@" , i)
        
  
  #Cleaning Mileage Column
  mileage = list(data.Mileage)
  
  for i in range(len(mileage)):
    try :
      mileage[i] = float(mileage[i].split(" ")[0].strip())
    except:
      mileage[i] = np.nan
      
  #Cleaning Engine Column   
  engine = list(data.Engine)
  for i in range(len(engine)):
    try :
      engine[i] = int(engine[i].split(" ")[0].strip())
    except:
      engine[i] = np.nan
      
  #Cleaning Power Columns
  power = list(data.Power)
  for i in range(len(power)):
    try :
      power[i] = float(power[i].split(" ")[0].strip())
    except:
      power[i] = np.nan
      
  #Cleaning New_Price
  data['New_Price'].fillna(0, inplace = True)
  
  newp = list(data['New_Price'])
  
  for i in range(len(newp)):
    if newp[i] == 0:
      newp[i] = float(newp[i])
      continue
    elif 'Cr' in newp[i]:
      newp[i] = float(newp[i].split()[0].strip()) * 100 
    elif 'Lakh' in newp[i]:
      newp[i] = float(newp[i].split()[0].strip())
      
      
  #Re-ordering the columns

  restructured = pd.DataFrame({'Brand': brand,
                              'Model':model,
                              'Location': data['Location'], 
                              'Year':data['Year'] , 
                              'Kilometers_Driven':data['Kilometers_Driven'],
                              'Fuel_Type':data['Fuel_Type'],
                              'Transmission':data['Transmission'],
                              'Owner_Type':data['Owner_Type'],
                              'Mileage':mileage,
                              'Engine':engine,
                              'Power':power,
                              'Seats':data['Seats'],
                              'New_Price':newp
                             })

  #If the dataset passed is training set include the Price column
  if 'Price' in data.columns:
    restructured['Price'] = data['Price']
    return restructured
  
  
  else:
    return restructured

"""**Summary:**

The data is is restructured in the following ways:
1. The Name column in the original dataset is split in to two features, Brand and Model.
1. The Mileage column is cleaned to have float values.
1. The Engine column is cleaned to have integer values.
2. The Power column is cleaned to have integer values.
2. The New_Price column is cleaned to remove nulls and correct the units.
"""

#Restructuring Training and Test sets
train_data = restructure(training_set)
test_data = restructure(test_set)

#the dimensions of the training set
train_data.shape

#the dimensions of the test set
test_data.shape

#Top 5 rows of the training set
train_data.head(5)

#Top 5 rows of the test set
test_data.head()

"""## Regression With Fast.ai

###Creating A TabularList

TabularList in fastai is the basic ItemList for any kind of tabular data.It is a class to create a list of inputs in items for tabular data. 

Main Arguments:

cat_names : The categorical features in the data.

cont_names : The continuous features in the data.

procs : A liat of transformations to be applies to the data such as FillMissing, Categorify, Normalize etc.
"""

#Defining the keyword arguments for fastai's TabularList

#Path / default location for saving/loading models
path = ''

#The dependent variable/target
dep_var = 'Price'

#The list of categorical features in the dataset
cat_names = ['Brand', 'Model', 'Location', 'Fuel_Type', 'Transmission', 'Owner_Type'] 

#The list of continuous features in the dataset
#Exclude the Dependent variable 'Price'
cont_names =['Year', 'Kilometers_Driven', 'Mileage', 'Engine', 'Power', 'Seats', 'New_Price'] 

#List of Processes/transforms to be applied to the dataset
procs = [FillMissing, Categorify, Normalize]

#Start index for creating a validation set from train_data
start_indx = len(train_data) - int(len(train_data) * 0.2)

#End index for creating a validation set from train_data
end_indx = len(train_data)


#TabularList for Validation
val = (TabularList.from_df(train_data.iloc[start_indx:end_indx].copy(), path=path, cat_names=cat_names, cont_names=cont_names))

test = (TabularList.from_df(test_data, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs))

#TabularList for training
data = (TabularList.from_df(train_data, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)
                           .split_by_idx(list(range(start_indx,end_indx)))
                           .label_from_df(cols=dep_var)
                           .add_test(test)
                           .databunch())

"""**Summary:**

1. Initializing/Setting The parameters for TabularList such as path, dep_var, cat_names, cont_names and procs.
1. Setting the index for Validation set. The start index and End index are set in such a away that it takes the last 20% data from the training set for validation.
2. Creating TabularList for Validation set from train_data. 
2. Creating TabularList for Test set from test_data. 
1. Creating a DataBunch for the network.DataBunch is a class that binds train_dl,valid_dl and test_dl in a data object.
"""

#Display the data batch
data.show_batch(rows = 10)

"""###Initializing Neural Network"""

#Initializing the network
learn = tabular_learner(data, layers=[300,200, 100, 50], metrics= [rmse,r2_score])

"""The above line of code will initialize a neural network with 4 layers and the number of nodes in each layer as 300,200, 100 and 50 respectively. 

The network will use two primary metrics for evaluation:

* Root Mean Squared Error(RMSE)
* R-Squared
"""

#Show the complete Summary of the model
learn.summary

"""###Training The Network"""

learn.lr_find(start_lr = 1e-05,end_lr = 1e+05, num_it = 100)
learn.recorder.plot()

"""Learning rate is a hyper-parameter that controls how much the weights of the network is being adjusted with respect the loss gradient.

The lr_find method helps explore the learning rate in a specified range. The graph shows the deviation in loss with respect to the learning rate.
"""

#Fitting data and training the network
learn.fit_one_cycle(25)

"""**The above line trains the network for 25 epochs.**

### Evaluating Performance
"""

#Display Predictions On Training Data
learn.show_results(ds_type=DatasetType.Train,rows = 5)

#Display Predictions On Validation Data
learn.show_results(ds_type=DatasetType.Valid)

#Getting The Training And Validation Errors

tr = learn.validate(learn.data.train_dl)
va = learn.validate(learn.data.valid_dl)
print("The Metrics used In Evaluating The Network:", str(learn.metrics))

print("\nThe calculated RMSE & R-Squared For The Training Set :", tr[1:])
print("\nThe calculated RMSE & R-Squared For The Validation Set :", va[1:])

"""Summary:

The Root Mean Squared Error is the standard deviation of the errors/residuals. It tells us the 'Goodness Of Fit' of a model. The lower the value of RMSE the better the model.

The R-Squared metric also called the coefficient of determination is used to understand the variation in the dependent variable(y) and the independent variable(X).The closer the value of R-Squared is to one, the better the model.

**The above output suggests that:**

**The model/network was able to attain an RMSE of 1.4678 and an R_squared of 0.9726 while training and an RMSE of 3.1737 and an R_squared of 0.9107 while Validating on the validation set.**
"""

#Plotting The losses for training and validation
learn.recorder.plot_losses()

"""The above graph shows the change is loss during the course of training the network. At the beginning of the training we can see a high loss value. As the networks learned from the data, the loss started to drop until it could no longer improve during the course of training.

The validation shows a relatively consistent and low loss values. 


**Note :**

The validation losses are only calculated once per epoch, whereas training losses are calculated after
"""

#Plotting Momentum & Learning Rate
learn.recorder.plot_lr(show_moms=True)

"""The above plots learning rate and momentum during the course of training."""

#Plotting the metrics of evaluation
learn.recorder.plot_metrics()

"""The decreasing RMSE and increasing R-Squared depicts the Goodness Of Fit.

### Exploring Validation Predictions
"""

val = train_data.tail(1203)

#Converting the prediction to DataFrame for Comparing
val_preds = learn.get_preds(ds_type=DatasetType.Valid)[0]
val_preds = [i[0] for i in val_preds.tolist()] 
val['Predicted'] = val_preds

val.head()

"""#### Calculating RMLSE For Validation Predictions

Since the metric used in the hackathon for evaluating the predictions is RMSLE , we will calculate te same for the validation predictions to evaluate our model.
"""

import numpy as np
Y_true = val['Price']
pred = val['Predicted']


#RMSLE
error = np.square(np.log10(pred + 1) - np.log10(Y_true +1)).mean() ** 0.5

score = 1 - error
print("SCORE For Validation : ",score)

"""#### A Simple Analysis On Predictions"""

#Plotting The Average Price For A Given Car Brand, -- Actual vs Predicted

import matplotlib.pyplot as plt

plt.figure(figsize=(30, 3))
plt.plot(val.groupby(['Brand']).mean()['Price'], linewidth = 3, )

plt.plot(val.groupby(['Brand']).mean()['Predicted'],linewidth = 5, ls  = '--')
plt.title('Average Price By Brands')
plt.xlabel('Brands')
plt.ylabel('Price In Lacs')
plt.legend()
plt.show()

"""The above graph shows comparison of the the average actual price by Brand and the predicted price."""

print("R-Squared For Validation Set : ", r2_score(learn.get_preds(ds_type=DatasetType.Valid)[0], learn.get_preds(ds_type=DatasetType.Valid)[1]))

print("\nRMSE For Validation Set : ",root_mean_squared_error(learn.get_preds(ds_type=DatasetType.Valid)[0], learn.get_preds(ds_type=DatasetType.Valid)[1]))

"""###Predicting For Test Data

####Predicting For A Single Row OF Test Set
"""

#Test set data for row 0
test_data.iloc[0]

#Prediction in float for Test set data for row 0
float(learn.predict(test_data.iloc[0])[1])

"""####Predicting For Test Set"""

test_predictions = learn.get_preds(ds_type=DatasetType.Test)[0]

#Converting the tensor output to a list of predicted values
test_predictions = [i[0] for i in test_predictions.tolist()]

#Converting the prediction to . a dataframe
test_predictions = pd.DataFrame(test_predictions, columns = ['Price'])

#Writing the predictions to an excel file.
predictions.to_excel("Fast_ai_solution.xlsx", index = False)

"""**Submit the above file [here](https://www.machinehack.com/course/predicting-the-costs-of-used-cars-hackathon-by-imarticus/leaderboard) to find out your score. Good Luck!**"""
<filename>A_HANDS-ON_GUIDE_TO_REGRESSION_WITH_FASTAI.py
# -*- coding: utf-8 -*-
"""Published-Regression_Using_Fastai.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/17VOQ78Wwj2ByP98M6ojhqpkq0usp3Udt

#Regression With Fastai

---

Published at Analytics India Magazine -- [A HANDS-ON GUIDE TO REGRESSION WITH FAST.AI](https://analyticsindiamag.com/a-hands-on-guide-to-regression-with-fast-ai/)

**Getting Started With Regression**

Regression With Fast.ai in 7 simple steps:

* Importing the libraries
* Creating a TabularList
* Initialising Neural Network
* Training the model
* Evaluating the model
* A simple analysis on the predictions of the validation set
* Predicting using the network

##Importing All Major Libraries
"""

import pandas as pd
import numpy as np
from fastai.tabular import *

"""**The fastai.tabular package includes all the modules that are necessary for processing tabular data.**

## Importing The Data
"""

#Reading the datasets from excel sheet
training_set = pd.read_excel("Data_Train.xlsx")
test_set = pd.read_excel("Data_Test.xlsx")

"""## Understanding The Data"""

training_set.head(5)

# Checking the number of rows
print("\n\nNumber of observations in the datasets :\n",'#' * 40)

print("\nTraining Set : ",len(training_set))
print("Test Set : ",len(test_set))

# checking the number of features in the Datasets
print("\n\nNumber of features in the datasets :\n",'#' * 40)

print("\nTraining Set : ",len(training_set.columns))
print("Test Set : ",len(test_set.columns))

# checking the features in the Datasets
print("\n\nFeatures in the datasets :\n",'#' * 40)

print("\nTraining Set : ", list(training_set.columns))
print("Test Set : ",list(test_set.columns))

# Checking the data types of features
print("\n\nDatatypes of features in the datasets :\n",'#' * 40)

print("\nTraining Set : ", training_set.dtypes)
print("\nTest Set : ",test_set.dtypes)

# checking for NaNs or empty cells
print("\n\nEmpty cells or Nans in the datasets :\n",'#' * 40)

print("\nTraining Set : ",training_set.isnull().values.any())
print("\nTest Set : ",test_set.isnull().values.any())

# checking for NaNs or empty cells by column
print("\n\nNumber of empty cells or Nans in the datasets :\n",'#' * 40)

print("\nTraining Set : ","\n", training_set.isnull().sum())
print("\nTest Set : ",test_set.isnull().sum())

#Displaying dataset information
print("\n\nInfo:\n",'#' * 40)

training_set.info()

"""### Exploring Categorical features"""

# Non categorical Features in The dataset
training_set.select_dtypes(['int','float']).columns

#Categotical Features in The Dataset
training_set.select_dtypes('object').columns

#The Unique values in each of the categorical features 

all_brands = list(training_set.Name) + list(test_set.Name)

all_locations = list(training_set.Location) + list(test_set.Location)

all_fuel_types = list(training_set.Fuel_Type) + list(test_set.Fuel_Type)

all_transmissions = list(training_set.Transmission) + list(test_set.Transmission)

all_owner_types = list(training_set.Owner_Type) + list(test_set.Owner_Type)



print("\nNumber Of Unique Values In Name : \n ", len(set(all_brands)))
#print("\nThe Unique Values In Name : \n ", set(all_brands))

print("\nNumber Of Unique Values In Location : \n ", len(set(all_locations)))
print("\nThe Unique Values In Location : \n ", set(all_locations) )

print("\nNumber Of Unique Values In Fuel_Type : \n ", len(set(all_fuel_types)))
print("\nThe Unique Values In Fuel_Type : \n ", set(all_fuel_types) )

print("\nNumber Of Unique Values In Transmission : \n ", len(set(all_transmissions)))
print("\nThe Unique Values In Transmission : \n ", set(all_transmissions) )

print("\nNumber Of Unique Values In Owner_Type : \n ", len(set(all_owner_types)))
print("\nThe Unique Values In Owner_Type : \n ", set(all_owner_types) )

"""## Feature Generation And Dataset Restructuring"""

#Based on the information gathered from the data, lets simplify and restructure it.

def restructure(data):
  
  names = list(data.Name)
  
  brand = []
  model = []
  
  #Splitting The Column 'Name'
  for i in range(len(names)):
    try:
      brand.append(names[i].split(" ")[0])
      try:
        model.append(" ".join(names[i].split(" ")[1:]).strip())
      except:
        pass
    except:
        print("ERR ! - ", names[i], "@" , i)
        
  
  #Cleaning Mileage Column
  mileage = list(data.Mileage)
  
  for i in range(len(mileage)):
    try :
      mileage[i] = float(mileage[i].split(" ")[0].strip())
    except:
      mileage[i] = np.nan
      
  #Cleaning Engine Column   
  engine = list(data.Engine)
  for i in range(len(engine)):
    try :
      engine[i] = int(engine[i].split(" ")[0].strip())
    except:
      engine[i] = np.nan
      
  #Cleaning Power Columns
  power = list(data.Power)
  for i in range(len(power)):
    try :
      power[i] = float(power[i].split(" ")[0].strip())
    except:
      power[i] = np.nan
      
  #Cleaning New_Price
  data['New_Price'].fillna(0, inplace = True)
  
  newp = list(data['New_Price'])
  
  for i in range(len(newp)):
    if newp[i] == 0:
      newp[i] = float(newp[i])
      continue
    elif 'Cr' in newp[i]:
      newp[i] = float(newp[i].split()[0].strip()) * 100 
    elif 'Lakh' in newp[i]:
      newp[i] = float(newp[i].split()[0].strip())
      
      
  #Re-ordering the columns

  restructured = pd.DataFrame({'Brand': brand,
                              'Model':model,
                              'Location': data['Location'], 
                              'Year':data['Year'] , 
                              'Kilometers_Driven':data['Kilometers_Driven'],
                              'Fuel_Type':data['Fuel_Type'],
                              'Transmission':data['Transmission'],
                              'Owner_Type':data['Owner_Type'],
                              'Mileage':mileage,
                              'Engine':engine,
                              'Power':power,
                              'Seats':data['Seats'],
                              'New_Price':newp
                             })

  #If the dataset passed is training set include the Price column
  if 'Price' in data.columns:
    restructured['Price'] = data['Price']
    return restructured
  
  
  else:
    return restructured

"""**Summary:**

The data is is restructured in the following ways:
1. The Name column in the original dataset is split in to two features, Brand and Model.
1. The Mileage column is cleaned to have float values.
1. The Engine column is cleaned to have integer values.
2. The Power column is cleaned to have integer values.
2. The New_Price column is cleaned to remove nulls and correct the units.
"""

#Restructuring Training and Test sets
train_data = restructure(training_set)
test_data = restructure(test_set)

#the dimensions of the training set
train_data.shape

#the dimensions of the test set
test_data.shape

#Top 5 rows of the training set
train_data.head(5)

#Top 5 rows of the test set
test_data.head()

"""## Regression With Fast.ai

###Creating A TabularList

TabularList in fastai is the basic ItemList for any kind of tabular data.It is a class to create a list of inputs in items for tabular data. 

Main Arguments:

cat_names : The categorical features in the data.

cont_names : The continuous features in the data.

procs : A liat of transformations to be applies to the data such as FillMissing, Categorify, Normalize etc.
"""

#Defining the keyword arguments for fastai's TabularList

#Path / default location for saving/loading models
path = ''

#The dependent variable/target
dep_var = 'Price'

#The list of categorical features in the dataset
cat_names = ['Brand', 'Model', 'Location', 'Fuel_Type', 'Transmission', 'Owner_Type'] 

#The list of continuous features in the dataset
#Exclude the Dependent variable 'Price'
cont_names =['Year', 'Kilometers_Driven', 'Mileage', 'Engine', 'Power', 'Seats', 'New_Price'] 

#List of Processes/transforms to be applied to the dataset
procs = [FillMissing, Categorify, Normalize]

#Start index for creating a validation set from train_data
start_indx = len(train_data) - int(len(train_data) * 0.2)

#End index for creating a validation set from train_data
end_indx = len(train_data)


#TabularList for Validation
val = (TabularList.from_df(train_data.iloc[start_indx:end_indx].copy(), path=path, cat_names=cat_names, cont_names=cont_names))

test = (TabularList.from_df(test_data, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs))

#TabularList for training
data = (TabularList.from_df(train_data, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)
                           .split_by_idx(list(range(start_indx,end_indx)))
                           .label_from_df(cols=dep_var)
                           .add_test(test)
                           .databunch())

"""**Summary:**

1. Initializing/Setting The parameters for TabularList such as path, dep_var, cat_names, cont_names and procs.
1. Setting the index for Validation set. The start index and End index are set in such a away that it takes the last 20% data from the training set for validation.
2. Creating TabularList for Validation set from train_data. 
2. Creating TabularList for Test set from test_data. 
1. Creating a DataBunch for the network.DataBunch is a class that binds train_dl,valid_dl and test_dl in a data object.
"""

#Display the data batch
data.show_batch(rows = 10)

"""###Initializing Neural Network"""

#Initializing the network
learn = tabular_learner(data, layers=[300,200, 100, 50], metrics= [rmse,r2_score])

"""The above line of code will initialize a neural network with 4 layers and the number of nodes in each layer as 300,200, 100 and 50 respectively. 

The network will use two primary metrics for evaluation:

* Root Mean Squared Error(RMSE)
* R-Squared
"""

#Show the complete Summary of the model
learn.summary

"""###Training The Network"""

learn.lr_find(start_lr = 1e-05,end_lr = 1e+05, num_it = 100)
learn.recorder.plot()

"""Learning rate is a hyper-parameter that controls how much the weights of the network is being adjusted with respect the loss gradient.

The lr_find method helps explore the learning rate in a specified range. The graph shows the deviation in loss with respect to the learning rate.
"""

#Fitting data and training the network
learn.fit_one_cycle(25)

"""**The above line trains the network for 25 epochs.**

### Evaluating Performance
"""

#Display Predictions On Training Data
learn.show_results(ds_type=DatasetType.Train,rows = 5)

#Display Predictions On Validation Data
learn.show_results(ds_type=DatasetType.Valid)

#Getting The Training And Validation Errors

tr = learn.validate(learn.data.train_dl)
va = learn.validate(learn.data.valid_dl)
print("The Metrics used In Evaluating The Network:", str(learn.metrics))

print("\nThe calculated RMSE & R-Squared For The Training Set :", tr[1:])
print("\nThe calculated RMSE & R-Squared For The Validation Set :", va[1:])

"""Summary:

The Root Mean Squared Error is the standard deviation of the errors/residuals. It tells us the 'Goodness Of Fit' of a model. The lower the value of RMSE the better the model.

The R-Squared metric also called the coefficient of determination is used to understand the variation in the dependent variable(y) and the independent variable(X).The closer the value of R-Squared is to one, the better the model.

**The above output suggests that:**

**The model/network was able to attain an RMSE of 1.4678 and an R_squared of 0.9726 while training and an RMSE of 3.1737 and an R_squared of 0.9107 while Validating on the validation set.**
"""

#Plotting The losses for training and validation
learn.recorder.plot_losses()

"""The above graph shows the change is loss during the course of training the network. At the beginning of the training we can see a high loss value. As the networks learned from the data, the loss started to drop until it could no longer improve during the course of training.

The validation shows a relatively consistent and low loss values. 


**Note :**

The validation losses are only calculated once per epoch, whereas training losses are calculated after
"""

#Plotting Momentum & Learning Rate
learn.recorder.plot_lr(show_moms=True)

"""The above plots learning rate and momentum during the course of training."""

#Plotting the metrics of evaluation
learn.recorder.plot_metrics()

"""The decreasing RMSE and increasing R-Squared depicts the Goodness Of Fit.

### Exploring Validation Predictions
"""

val = train_data.tail(1203)

#Converting the prediction to DataFrame for Comparing
val_preds = learn.get_preds(ds_type=DatasetType.Valid)[0]
val_preds = [i[0] for i in val_preds.tolist()] 
val['Predicted'] = val_preds

val.head()

"""#### Calculating RMLSE For Validation Predictions

Since the metric used in the hackathon for evaluating the predictions is RMSLE , we will calculate te same for the validation predictions to evaluate our model.
"""

import numpy as np
Y_true = val['Price']
pred = val['Predicted']


#RMSLE
error = np.square(np.log10(pred + 1) - np.log10(Y_true +1)).mean() ** 0.5

score = 1 - error
print("SCORE For Validation : ",score)

"""#### A Simple Analysis On Predictions"""

#Plotting The Average Price For A Given Car Brand, -- Actual vs Predicted

import matplotlib.pyplot as plt

plt.figure(figsize=(30, 3))
plt.plot(val.groupby(['Brand']).mean()['Price'], linewidth = 3, )

plt.plot(val.groupby(['Brand']).mean()['Predicted'],linewidth = 5, ls  = '--')
plt.title('Average Price By Brands')
plt.xlabel('Brands')
plt.ylabel('Price In Lacs')
plt.legend()
plt.show()

"""The above graph shows comparison of the the average actual price by Brand and the predicted price."""

print("R-Squared For Validation Set : ", r2_score(learn.get_preds(ds_type=DatasetType.Valid)[0], learn.get_preds(ds_type=DatasetType.Valid)[1]))

print("\nRMSE For Validation Set : ",root_mean_squared_error(learn.get_preds(ds_type=DatasetType.Valid)[0], learn.get_preds(ds_type=DatasetType.Valid)[1]))

"""###Predicting For Test Data

####Predicting For A Single Row OF Test Set
"""

#Test set data for row 0
test_data.iloc[0]

#Prediction in float for Test set data for row 0
float(learn.predict(test_data.iloc[0])[1])

"""####Predicting For Test Set"""

test_predictions = learn.get_preds(ds_type=DatasetType.Test)[0]

#Converting the tensor output to a list of predicted values
test_predictions = [i[0] for i in test_predictions.tolist()]

#Converting the prediction to . a dataframe
test_predictions = pd.DataFrame(test_predictions, columns = ['Price'])

#Writing the predictions to an excel file.
predictions.to_excel("Fast_ai_solution.xlsx", index = False)

"""**Submit the above file [here](https://www.machinehack.com/course/predicting-the-costs-of-used-cars-hackathon-by-imarticus/leaderboard) to find out your score. Good Luck!**"""
# -*- coding: utf-8 -*- Published-Regression_Using_Fastai.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/17VOQ78Wwj2ByP98M6ojhqpkq0usp3Udt  #Regression With Fastai  ---  Published at Analytics India Magazine -- [A HANDS-ON GUIDE TO REGRESSION WITH FAST.AI](https://analyticsindiamag.com/a-hands-on-guide-to-regression-with-fast-ai/)  **Getting Started With Regression**  Regression With Fast.ai in 7 simple steps:  * Importing the libraries * Creating a TabularList * Initialising Neural Network * Training the model * Evaluating the model * A simple analysis on the predictions of the validation set * Predicting using the network  ##Importing All Major Libraries **The fastai.tabular package includes all the modules that are necessary for processing tabular data.**  ## Importing The Data #Reading the datasets from excel sheet ## Understanding The Data # Checking the number of rows # checking the number of features in the Datasets # checking the features in the Datasets # Checking the data types of features # checking for NaNs or empty cells # checking for NaNs or empty cells by column #Displaying dataset information ### Exploring Categorical features # Non categorical Features in The dataset #Categotical Features in The Dataset #The Unique values in each of the categorical features #print("\nThe Unique Values In Name : \n ", set(all_brands)) ## Feature Generation And Dataset Restructuring #Based on the information gathered from the data, lets simplify and restructure it. #Splitting The Column 'Name' #Cleaning Mileage Column #Cleaning Engine Column #Cleaning Power Columns #Cleaning New_Price #Re-ordering the columns #If the dataset passed is training set include the Price column **Summary:**  The data is is restructured in the following ways: 1. The Name column in the original dataset is split in to two features, Brand and Model. 1. The Mileage column is cleaned to have float values. 1. The Engine column is cleaned to have integer values. 2. The Power column is cleaned to have integer values. 2. The New_Price column is cleaned to remove nulls and correct the units. #Restructuring Training and Test sets #the dimensions of the training set #the dimensions of the test set #Top 5 rows of the training set #Top 5 rows of the test set ## Regression With Fast.ai  ###Creating A TabularList  TabularList in fastai is the basic ItemList for any kind of tabular data.It is a class to create a list of inputs in items for tabular data.   Main Arguments:  cat_names : The categorical features in the data.  cont_names : The continuous features in the data.  procs : A liat of transformations to be applies to the data such as FillMissing, Categorify, Normalize etc. #Defining the keyword arguments for fastai's TabularList #Path / default location for saving/loading models #The dependent variable/target #The list of categorical features in the dataset #The list of continuous features in the dataset #Exclude the Dependent variable 'Price' #List of Processes/transforms to be applied to the dataset #Start index for creating a validation set from train_data #End index for creating a validation set from train_data #TabularList for Validation #TabularList for training **Summary:**  1. Initializing/Setting The parameters for TabularList such as path, dep_var, cat_names, cont_names and procs. 1. Setting the index for Validation set. The start index and End index are set in such a away that it takes the last 20% data from the training set for validation. 2. Creating TabularList for Validation set from train_data.  2. Creating TabularList for Test set from test_data.  1. Creating a DataBunch for the network.DataBunch is a class that binds train_dl,valid_dl and test_dl in a data object. #Display the data batch ###Initializing Neural Network #Initializing the network The above line of code will initialize a neural network with 4 layers and the number of nodes in each layer as 300,200, 100 and 50 respectively.   The network will use two primary metrics for evaluation:  * Root Mean Squared Error(RMSE) * R-Squared #Show the complete Summary of the model ###Training The Network Learning rate is a hyper-parameter that controls how much the weights of the network is being adjusted with respect the loss gradient.  The lr_find method helps explore the learning rate in a specified range. The graph shows the deviation in loss with respect to the learning rate. #Fitting data and training the network **The above line trains the network for 25 epochs.**  ### Evaluating Performance #Display Predictions On Training Data #Display Predictions On Validation Data #Getting The Training And Validation Errors Summary:  The Root Mean Squared Error is the standard deviation of the errors/residuals. It tells us the 'Goodness Of Fit' of a model. The lower the value of RMSE the better the model.  The R-Squared metric also called the coefficient of determination is used to understand the variation in the dependent variable(y) and the independent variable(X).The closer the value of R-Squared is to one, the better the model.  **The above output suggests that:**  **The model/network was able to attain an RMSE of 1.4678 and an R_squared of 0.9726 while training and an RMSE of 3.1737 and an R_squared of 0.9107 while Validating on the validation set.** #Plotting The losses for training and validation The above graph shows the change is loss during the course of training the network. At the beginning of the training we can see a high loss value. As the networks learned from the data, the loss started to drop until it could no longer improve during the course of training.  The validation shows a relatively consistent and low loss values.    **Note :**  The validation losses are only calculated once per epoch, whereas training losses are calculated after #Plotting Momentum & Learning Rate The above plots learning rate and momentum during the course of training. #Plotting the metrics of evaluation The decreasing RMSE and increasing R-Squared depicts the Goodness Of Fit.  ### Exploring Validation Predictions #Converting the prediction to DataFrame for Comparing #### Calculating RMLSE For Validation Predictions  Since the metric used in the hackathon for evaluating the predictions is RMSLE , we will calculate te same for the validation predictions to evaluate our model. #RMSLE #### A Simple Analysis On Predictions #Plotting The Average Price For A Given Car Brand, -- Actual vs Predicted The above graph shows comparison of the the average actual price by Brand and the predicted price. ###Predicting For Test Data  ####Predicting For A Single Row OF Test Set #Test set data for row 0 #Prediction in float for Test set data for row 0 ####Predicting For Test Set #Converting the tensor output to a list of predicted values #Converting the prediction to . a dataframe #Writing the predictions to an excel file. **Submit the above file [here](https://www.machinehack.com/course/predicting-the-costs-of-used-cars-hackathon-by-imarticus/leaderboard) to find out your score. Good Luck!**