<gh_stars>0
import pandas as pd
import numpy as np

train = pd.read_csv("Train.csv",names=['Item_Identifier', 'Item_Weight', 'Item_Fat_Content','Item_Visibility', 'Item_Type', 'Item_MRP', 'Outlet_Identifier', 'Outlet_Establishment_Year', 'Outlet_Size', 'Outlet_Location_Type', 'Outlet_Type', 'Item_Outlet_Sales'])
test = pd.read_csv("Test.csv",names=['Item_Identifier', 'Item_Weight', 'Item_Fat_Content','Item_Visibility', 'Item_Type', 'Item_MRP', 'Outlet_Identifier', 'Outlet_Establishment_Year', 'Outlet_Size', 'Outlet_Location_Type', 'Outlet_Type', 'Item_Outlet_Sales'])
train['source']='train'
test['source']='test'
data = pd.concat([train, test],ignore_index=True)
data.apply(lambda x: sum(x.isnull()))
data.apply(lambda x: len(x.unique()))
categorical_columns = [x for x in data.dtypes.index if data.dtypes[x]=='object']
categorical_columns = [x for x in categorical_columns if x not in ['Item_Identifier','Outlet_Identifier','source']]

#1
item_avg_weight = data.pivot_table(values='Item_Weight', index='Item_Identifier')
data.set_index('Item_Identifier',inplace=True)
data['Item_Weight'].fillna(item_avg_weight.Item_Weight,inplace=True)
# print('Initial #missing: %d'% sum(data['Item_Weight'].isnull()))
data['Item_Weight']=data['Item_Weight'].replace(np.nan,data['Item_Weight'].mean())
data.reset_index(inplace=True)
index = data['Item_Weight'].index[data['Item_Weight'].apply(np.isnan)]
# print(index)
# print('Final #missing: %d'% sum(data['Item_Weight'].isnull()))

# #2
data['Outlet_Size'].fillna(data['Outlet_Size'].mode()[0],inplace=True)
# data['Outlet_Size']=data.groupby(['Item_Identifier']).agg(lambda x:x.value_counts().index[0])
#
# #3
miss_bool = data['Item_Visibility'].isnull()
# # # print('initial : ',sum(miss_bool))
visibility_avg = data.pivot_table(values='Item_Visibility', index='Item_Identifier')
# print(visibility_avg)
data.set_index('Item_Identifier',inplace=True)
data['Item_Visibility'].replace(0,visibility_avg.Item_Visibility,inplace=True)
data.reset_index(inplace=True)
# # print('Number of 0 values after modification: %d'%sum(data['Item_Visibility'] == 0))
# # print(data['Item_Visibility'])
# #
# # #4

data.set_index('Item_Identifier',inplace=True)
data['Item_Visibility_MeanRatio']=np.zeros(len(data.index))
data['Item_Visibility_MeanRatio'].replace(0,data['Item_Visibility']/visibility_avg.Item_Visibility,inplace=True)
data.reset_index(inplace=True)
# data['Item_Visibility_MeanRatio'] = data.apply(lambda x: x['Item_Visibility']/visibility_avg[x['Item_Identifier']], axis=1)
# print(data['Item_Visibility_MeanRatio'].head())

#5
data['Item_Type_Combined'] = data['Item_Identifier'].apply(lambda x: x[0:2])
#Rename them to more intuitive categories:
data['Item_Type_Combined'] = data['Item_Type_Combined'].map({'FD':'Food','NC':'Non-Consumable','DR':'Drinks'})
data['Item_Type_Combined'].value_counts()

data['Outlet_Years'] = 2013 - data['Outlet_Establishment_Year']
# print(data['Outlet_Years'].describe())

#6
data['Item_Fat_Content'] = data['Item_Fat_Content'].replace({'LF':'Low Fat','reg':'Regular','low fat':'Low Fat'})
# print(data['Item_Fat_Content'].value_counts())

data.loc[data['Item_Type_Combined']=="Non-Consumable",'Item_Fat_Content'] = "Non-Edible"
data['Item_Fat_Content'].value_counts()

#7 One hot encoding

from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
#New variable for outlet
data['Outlet'] = le.fit_transform(data['Outlet_Identifier'])
var_mod = ['Item_Fat_Content','Outlet_Location_Type','Outlet_Size','Item_Type_Combined','Outlet_Type','Outlet']
le = LabelEncoder()
for i in var_mod:
    data[i] = le.fit_transform(data[i])
data = pd.get_dummies(data, columns=['Item_Fat_Content','Outlet_Location_Type','Outlet_Size','Outlet_Type','Item_Type_Combined','Outlet'])
# print(data[['Item_Fat_Content_0','Item_Fat_Content_1','Item_Fat_Content_2']].head(10))

#8

data.drop(['Item_Type','Outlet_Establishment_Year'],axis=1,inplace=True)

#Divide into test and train:
train = data.loc[data['source']=="train"]
test = data.loc[data['source']=="test"]

#Drop unnecessary columns:
test.drop(['Item_Outlet_Sales','source'],axis=1,inplace=True)
train.drop(['source'],axis=1,inplace=True)
#Export files as modified versions:
train.to_csv("train_modified.csv",index=False)
test.to_csv("test_modified.csv",index=False)

mean_sales = train['Item_Outlet_Sales'].mean()
#Define a dataframe with IDs for submission:
base1 = test[['Item_Identifier','Outlet_Identifier']]
base1['Item_Outlet_Sales'] = mean_sales
#Export submission file
base1.to_csv("alg0.csv",index=False)

#9

target = 'Item_Outlet_Sales'
IDcol = ['Item_Identifier', 'Outlet_Identifier']
from sklearn import cross_validation, metrics


def modelfit(alg, dtrain, dtest, predictors, target, IDcol, filename):
    # Fit the algorithm on the data
    alg.fit(dtrain[predictors], dtrain[target])

    # Predict training set:
    dtrain_predictions = alg.predict(dtrain[predictors])

    # Perform cross-validation:
    cv_score = cross_validation.cross_val_score(alg, dtrain[predictors], dtrain[target], cv=20,
                                                scoring='mean_squared_error')
    cv_score = np.sqrt(np.abs(cv_score))

    # Print model report:
    print("\nModel Report")
    print("RMSE : %.4g" % np.sqrt(metrics.mean_squared_error(dtrain[target].values, dtrain_predictions)))
    print("CV Score : Mean - %.4g | Std - %.4g | Min - %.4g | Max - %.4g" % (np.mean(cv_score), np.std(cv_score), np.min(cv_score), np.max(cv_score)))

    # Predict on testing data:
    dtest[target] = alg.predict(dtest[predictors])

    # Export submission file:
    IDcol.append(target)
    submission = pd.DataFrame({x: dtest[x] for x in IDcol})
    submission.to_csv(filename, index=True)

    #10

from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import ElasticNet

predictors = [x for x in train.columns if x not in [target] + IDcol]
# print predictors
# alg2 = RandomForestRegressor(n_estimators=400, max_depth=6, min_samples_leaf=100, n_jobs=4)
alg3= ElasticNet(alpha=1, l1_ratio=0.5, normalize=False)
modelfit(alg3, train, test, predictors, target, IDcol, 'alg3.csv')

import pandas as pd
import numpy as np

train = pd.read_csv("Train.csv",names=['Item_Identifier', 'Item_Weight', 'Item_Fat_Content','Item_Visibility', 'Item_Type', 'Item_MRP', 'Outlet_Identifier', 'Outlet_Establishment_Year', 'Outlet_Size', 'Outlet_Location_Type', 'Outlet_Type', 'Item_Outlet_Sales'])
test = pd.read_csv("Test.csv",names=['Item_Identifier', 'Item_Weight', 'Item_Fat_Content','Item_Visibility', 'Item_Type', 'Item_MRP', 'Outlet_Identifier', 'Outlet_Establishment_Year', 'Outlet_Size', 'Outlet_Location_Type', 'Outlet_Type', 'Item_Outlet_Sales'])
train['source']='train'
test['source']='test'
data = pd.concat([train, test],ignore_index=True)
data.apply(lambda x: sum(x.isnull()))
data.apply(lambda x: len(x.unique()))
categorical_columns = [x for x in data.dtypes.index if data.dtypes[x]=='object']
categorical_columns = [x for x in categorical_columns if x not in ['Item_Identifier','Outlet_Identifier','source']]

#1
item_avg_weight = data.pivot_table(values='Item_Weight', index='Item_Identifier')
data.set_index('Item_Identifier',inplace=True)
data['Item_Weight'].fillna(item_avg_weight.Item_Weight,inplace=True)
# print('Initial #missing: %d'% sum(data['Item_Weight'].isnull()))
data['Item_Weight']=data['Item_Weight'].replace(np.nan,data['Item_Weight'].mean())
data.reset_index(inplace=True)
index = data['Item_Weight'].index[data['Item_Weight'].apply(np.isnan)]
# print(index)
# print('Final #missing: %d'% sum(data['Item_Weight'].isnull()))

# #2
data['Outlet_Size'].fillna(data['Outlet_Size'].mode()[0],inplace=True)
# data['Outlet_Size']=data.groupby(['Item_Identifier']).agg(lambda x:x.value_counts().index[0])
#
# #3
miss_bool = data['Item_Visibility'].isnull()
# # # print('initial : ',sum(miss_bool))
visibility_avg = data.pivot_table(values='Item_Visibility', index='Item_Identifier')
# print(visibility_avg)
data.set_index('Item_Identifier',inplace=True)
data['Item_Visibility'].replace(0,visibility_avg.Item_Visibility,inplace=True)
data.reset_index(inplace=True)
# # print('Number of 0 values after modification: %d'%sum(data['Item_Visibility'] == 0))
# # print(data['Item_Visibility'])
# #
# # #4

data.set_index('Item_Identifier',inplace=True)
data['Item_Visibility_MeanRatio']=np.zeros(len(data.index))
data['Item_Visibility_MeanRatio'].replace(0,data['Item_Visibility']/visibility_avg.Item_Visibility,inplace=True)
data.reset_index(inplace=True)
# data['Item_Visibility_MeanRatio'] = data.apply(lambda x: x['Item_Visibility']/visibility_avg[x['Item_Identifier']], axis=1)
# print(data['Item_Visibility_MeanRatio'].head())

#5
data['Item_Type_Combined'] = data['Item_Identifier'].apply(lambda x: x[0:2])
#Rename them to more intuitive categories:
data['Item_Type_Combined'] = data['Item_Type_Combined'].map({'FD':'Food','NC':'Non-Consumable','DR':'Drinks'})
data['Item_Type_Combined'].value_counts()

data['Outlet_Years'] = 2013 - data['Outlet_Establishment_Year']
# print(data['Outlet_Years'].describe())

#6
data['Item_Fat_Content'] = data['Item_Fat_Content'].replace({'LF':'Low Fat','reg':'Regular','low fat':'Low Fat'})
# print(data['Item_Fat_Content'].value_counts())

data.loc[data['Item_Type_Combined']=="Non-Consumable",'Item_Fat_Content'] = "Non-Edible"
data['Item_Fat_Content'].value_counts()

#7 One hot encoding

from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
#New variable for outlet
data['Outlet'] = le.fit_transform(data['Outlet_Identifier'])
var_mod = ['Item_Fat_Content','Outlet_Location_Type','Outlet_Size','Item_Type_Combined','Outlet_Type','Outlet']
le = LabelEncoder()
for i in var_mod:
    data[i] = le.fit_transform(data[i])
data = pd.get_dummies(data, columns=['Item_Fat_Content','Outlet_Location_Type','Outlet_Size','Outlet_Type','Item_Type_Combined','Outlet'])
# print(data[['Item_Fat_Content_0','Item_Fat_Content_1','Item_Fat_Content_2']].head(10))

#8

data.drop(['Item_Type','Outlet_Establishment_Year'],axis=1,inplace=True)

#Divide into test and train:
train = data.loc[data['source']=="train"]
test = data.loc[data['source']=="test"]

#Drop unnecessary columns:
test.drop(['Item_Outlet_Sales','source'],axis=1,inplace=True)
train.drop(['source'],axis=1,inplace=True)
#Export files as modified versions:
train.to_csv("train_modified.csv",index=False)
test.to_csv("test_modified.csv",index=False)

mean_sales = train['Item_Outlet_Sales'].mean()
#Define a dataframe with IDs for submission:
base1 = test[['Item_Identifier','Outlet_Identifier']]
base1['Item_Outlet_Sales'] = mean_sales
#Export submission file
base1.to_csv("alg0.csv",index=False)

#9

target = 'Item_Outlet_Sales'
IDcol = ['Item_Identifier', 'Outlet_Identifier']
from sklearn import cross_validation, metrics


def modelfit(alg, dtrain, dtest, predictors, target, IDcol, filename):
    # Fit the algorithm on the data
    alg.fit(dtrain[predictors], dtrain[target])

    # Predict training set:
    dtrain_predictions = alg.predict(dtrain[predictors])

    # Perform cross-validation:
    cv_score = cross_validation.cross_val_score(alg, dtrain[predictors], dtrain[target], cv=20,
                                                scoring='mean_squared_error')
    cv_score = np.sqrt(np.abs(cv_score))

    # Print model report:
    print("\nModel Report")
    print("RMSE : %.4g" % np.sqrt(metrics.mean_squared_error(dtrain[target].values, dtrain_predictions)))
    print("CV Score : Mean - %.4g | Std - %.4g | Min - %.4g | Max - %.4g" % (np.mean(cv_score), np.std(cv_score), np.min(cv_score), np.max(cv_score)))

    # Predict on testing data:
    dtest[target] = alg.predict(dtest[predictors])

    # Export submission file:
    IDcol.append(target)
    submission = pd.DataFrame({x: dtest[x] for x in IDcol})
    submission.to_csv(filename, index=True)

    #10

from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import ElasticNet

predictors = [x for x in train.columns if x not in [target] + IDcol]
# print predictors
# alg2 = RandomForestRegressor(n_estimators=400, max_depth=6, min_samples_leaf=100, n_jobs=4)
alg3= ElasticNet(alpha=1, l1_ratio=0.5, normalize=False)
modelfit(alg3, train, test, predictors, target, IDcol, 'alg3.csv')
#1 # print('Initial #missing: %d'% sum(data['Item_Weight'].isnull())) # print(index) # print('Final #missing: %d'% sum(data['Item_Weight'].isnull())) # #2 # data['Outlet_Size']=data.groupby(['Item_Identifier']).agg(lambda x:x.value_counts().index[0]) # # #3 # # # print('initial : ',sum(miss_bool)) # print(visibility_avg) # # print('Number of 0 values after modification: %d'%sum(data['Item_Visibility'] == 0)) # # print(data['Item_Visibility']) # # # # #4 # data['Item_Visibility_MeanRatio'] = data.apply(lambda x: x['Item_Visibility']/visibility_avg[x['Item_Identifier']], axis=1) # print(data['Item_Visibility_MeanRatio'].head()) #5 #Rename them to more intuitive categories: # print(data['Outlet_Years'].describe()) #6 # print(data['Item_Fat_Content'].value_counts()) #7 One hot encoding #New variable for outlet # print(data[['Item_Fat_Content_0','Item_Fat_Content_1','Item_Fat_Content_2']].head(10)) #8 #Divide into test and train: #Drop unnecessary columns: #Export files as modified versions: #Define a dataframe with IDs for submission: #Export submission file #9 # Fit the algorithm on the data # Predict training set: # Perform cross-validation: # Print model report: # Predict on testing data: # Export submission file: #10 # print predictors # alg2 = RandomForestRegressor(n_estimators=400, max_depth=6, min_samples_leaf=100, n_jobs=4)