<a href="https://colab.research.google.com/github/ancastillar/Study-on-the-probability-of-payment-of-a-credit-requested-by-a-customer/blob/main/Model_LOAN_Random_Forest.ipynb" target="_parent"></a>


```
import pandas as pd
import pydotplus
import numpy  as np
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm

from io import StringIO
from IPython.display import Image, SVG
from graphviz import Source
from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from matplotlib.patches import Patch
from scipy.stats import chi2_contingency, norm
from sklearn.tree import export_graphviz
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve, auc, accuracy_score
from sklearn.model_selection import StratifiedKFold, train_test_split
from statsmodels.formula.api import ols
from sympy import var, plot_implicit, Eq
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_auc_score 



plt.style.use('ggplot')
# ignore log(0) and divide by 0 warning
np.seterr(divide='ignore');
SEED=1
```

## Decision trees and random forests



##Decision trees##

*   In contrast with linear models, like logistic regression, decision trees can capture non-linear relationship in data.
*   Futhermore, decision trees don't require the features to be on the same scale
* The decision region: region in the feauture space where all instances are assigned to one class label. Produces rectangular decision regions in the feature-space





```
df = pd.read_csv('/content/drive/MyDrive/data_davivienda/Lending_club.csv',
                 dtype = {'loan_status':'category', 'annual_inc':'float',
                          'verification_status':'category', 'emp_length':'category',
                          'home_ownership':'category', 'int_rate':'object',
                          'loan_amnt':'float', 'purpose':'category',
                          'term':'category', 'grade':'category'})

#Change loan_status, verification_status, emp_length, term and grade to category type 
df.loan_status = df.loan_status.astype(pd.api.types.CategoricalDtype(categories=['Charged Off', 'Fully Paid']))
df.verification_status = df.verification_status.astype(pd.api.types.CategoricalDtype(categories=['Not Verified', 'Source Verified', 'Verified']))
df.emp_length = df.emp_length.astype(pd.api.types.CategoricalDtype(categories=['< 1 year', '1 year', '2 years', '3 years', '4 years', \
                                                             '5 years', '6 years', '7 years', '8 years', '9 years', \
                                                             '10+ years']))
df.home_ownership = df.home_ownership.astype(pd.api.types.CategoricalDtype(categories=['RENT','MORTGAGE','OWN','OTHER']))
df.term = df.term.astype(pd.api.types.CategoricalDtype(categories=[' 36 months', ' 60 months']))
df.grade = df.grade.astype(pd.api.types.CategoricalDtype(categories=['A','B','C','D','E','F','G']))
df.purpose = df.purpose.astype(pd.api.types.CategoricalDtype(categories=['debt_consolidation','credit_card','other','home_improvement','major_purchase','small_business','car', 'wedding','medical','moving','house','vacation','educational','renewable_energy']))

#In addition, the original data in int_rate contains strings of the form 'x.xx%',
#we remove the % and change the vaules to float:
df.int_rate = df.int_rate.str.rstrip('%').astype('float')

```


```
df_t=df.copy()
for var in ['loan_status', 'term','verification_status','grade','home_ownership']:
    df_t[var]=df_t[var].cat.codes


clf = DecisionTreeClassifier(max_depth = 3)

# Train Decision Tree Classifer
covariates=['loan_amnt']

clf = clf.fit(df_t[covariates], df_t['loan_status'])

dot_data = StringIO()
export_graphviz(clf, out_file=dot_data, feature_names = covariates,
                class_names=['Charged Off', 'Fully Paid'],  
                filled=True, rounded=True,
                special_characters=True, 
                impurity='gini', proportion=True)

graph = pydotplus.graph_from_dot_data(dot_data.getvalue())  
Image(graph.create_png())
```

We need to determine which variable we are going to put at the beginning of the tree. The algorithm does that with the concept of gain information:


> $\text{IG(f, sp)}= \text{IG(parent)}- ( \frac{N_l}{N}\text{IG(left)}+\frac{N_r}{N}\text{IG(right)})$ 




##training##


```
df_pre = pd.concat([(df.loan_amnt - df.loan_amnt.mean())/df.loan_amnt.std(),\
                     (df.int_rate - df.int_rate.mean())/df.int_rate.std(),\
                     (df.annual_inc - df.annual_inc.mean())/df.annual_inc.std(),\
                     pd.get_dummies(df.home_ownership, prefix='home', drop_first=True), \
                     pd.get_dummies(df.purpose, prefix='purpose', drop_first=True), \
                     pd.get_dummies(df.grade, prefix='grade',drop_first=True)], axis=1)
df_pre['verification_status'] = df.verification_status.cat.codes
df_pre['emp_length'] = df.emp_length.cat.codes
df_pre['term'] = df.term.cat.codes
```


```
y=df.loan_status.cat.codes
X=df_pre
X_train, X_test, y_train, y_test= train_test_split(X, y,test_size=0.2, stratify=y,random_state=SEED)              
```


```
dt = DecisionTreeClassifier(random_state=SEED)                    
```


```
#params
params_dt = {'max_depth':[2,3,4,6], 'min_samples_leaf':[0.12, 0.14, 0.16, 0.18,0.25]}
```


```
# Instantiate grid_dt
grid_dt = GridSearchCV(estimator=dt,
                       param_grid=params_dt,
                       scoring='roc_auc',
                       cv=5,
                       n_jobs=-1)
#training
grid_dt.fit(X_train, y_train)
```




    GridSearchCV(cv=5, error_score=nan,
                 estimator=DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None,
                                                  criterion='gini', max_depth=None,
                                                  max_features=None,
                                                  max_leaf_nodes=None,
                                                  min_impurity_decrease=0.0,
                                                  min_impurity_split=None,
                                                  min_samples_leaf=1,
                                                  min_samples_split=2,
                                                  min_weight_fraction_leaf=0.0,
                                                  presort='deprecated',
                                                  random_state=1, splitter='best'),
                 iid='deprecated', n_jobs=-1,
                 param_grid={'max_depth': [2, 3, 4, 6],
                             'min_samples_leaf': [0.12, 0.14, 0.16, 0.18, 0.25]},
                 pre_dispatch='2*n_jobs', refit=True, return_train_score=False,
                 scoring='roc_auc', verbose=0)




```
# Extract the best estimator
best_model = grid_dt.best_estimator_
best_hyperparams = grid_dt.best_params_
print('Best hyerparameters:\n', best_hyperparams)

# Predict the test set probabilities of the positive class
y_pred_proba = best_model.predict_proba(X_test)[:,1]

# Compute test_roc_auc
test_roc_auc = roc_auc_score(y_test, y_pred_proba)

# Print test_roc_auc
print('Test set ROC AUC score: {:.3f}'.format(test_roc_auc))

```

    Best hyerparameters:
     {'max_depth': 4, 'min_samples_leaf': 0.12}
    Test set ROC AUC score: 0.661


### Decision boundary of a decision tree

The decision boundary of decision tree can be highly non-linear. It usually looks like patches due to the fact that we are dichotomizing the dataset at multiple levels. The following is an example for a tree built with `annual_inc` and `loan_amnt`:


```
n_classes = 2
plot_colors = 'br'

labels = ['Charged off', 'Fully paid']

clf = DecisionTreeClassifier(random_state=SEED).fit(df[['annual_inc', 'loan_amnt']], df['loan_status'].cat.codes)

# Plot the decision boundary

plot_step = 100
viridis = cm.get_cmap('viridis', 2)
newcolors = viridis(np.linspace(0, 1, 2))
blue = np.array([0, 0, 1, 1])
red = np.array([1, 0, 0, 1])
newcolors[0, :] = blue
newcolors[1, :] = red
newcmp = ListedColormap(newcolors)

np.random.seed(0)
idx_plot = np.random.choice(df.shape[0], 100, replace = False )
df_plot = df.iloc[idx_plot]

x_min, x_max = df_plot['annual_inc'].min() - 1000, df_plot['annual_inc'].max() + 1000
y_min, y_max = df_plot['loan_amnt'].min() - 1000, df_plot['loan_amnt'].max() + 1000
xx, yy = np.meshgrid(np.arange(x_min, x_max, plot_step),
                     np.arange(y_min, y_max, plot_step))

fig, ax = plt.subplots()
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
cs = plt.contourf(xx, yy, Z, cmap=newcmp, alpha = 0.7)

plt.xlabel('Annual income')
plt.ylabel('Loan amount')
plt.axis('tight');

for i, color in zip(range(n_classes), plot_colors):
    idx = np.where(df_plot['loan_status'].cat.codes == i)
    plt.scatter(df_plot.iloc[idx]['annual_inc'], df_plot.iloc[idx]['loan_amnt'], c=color, 
                label=labels[i], edgecolors='black')

legend_elements = [Patch(facecolor='blue', edgecolor='blue', label='Charged off'),
                   Patch(facecolor='red', edgecolor='red', label='Fully paid')]

plt.axis('tight')
plt.title('Decision surface of a decision tree')
leg1 = ax.legend(loc='upper left', title='Observed outcome')
ax.legend(handles=legend_elements, loc='lower right', title='Predicted outcome');
ax.add_artist(leg1);
```



*   The area of a single rectangule depends of hyperparameters of DT
*   The max_depth parameter defines whether the model raises a bias or variance problem



##Random Forest

The idea behind random forests is to construct a lot of decision trees, then use a voting mechanism to produce a final prediction that is agreed upon by most of the decision trees. When training these individual trees, we also subsample on the dataset as well as the covariates that are available in order to get as much built-in cross-validation as possible. As a result, the problem of overfitting and stability in a single decision tree model is resolved to a large degree.


```
rf = RandomForestClassifier(random_state=SEED)
```


```
#params
params_rf = {'max_features':['log2','sqrt'],'n_estimators':[300,400,500,800],'max_depth':[2,3,4,6], 'min_samples_leaf':[0.12, 0.14, 0.16, 0.18,0.25]}
```


```
# Instantiate grid_dt
grid_rf = GridSearchCV(estimator=rf,
                       param_grid=params_rf,
                       scoring='neg_mean_squared_error',
                       cv=5,
                       n_jobs=-1)
#training
grid_rf.fit(X_train, y_train)
```




    GridSearchCV(cv=5, error_score=nan,
                 estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,
                                                  class_weight=None,
                                                  criterion='gini', max_depth=None,
                                                  max_features='auto',
                                                  max_leaf_nodes=None,
                                                  max_samples=None,
                                                  min_impurity_decrease=0.0,
                                                  min_impurity_split=None,
                                                  min_samples_leaf=1,
                                                  min_samples_split=2,
                                                  min_weight_fraction_leaf=0.0,
                                                  n_estimators=100, n_jobs=None,
                                                  oob_score=False, random_state=1,
                                                  verbose=0, warm_start=False),
                 iid='deprecated', n_jobs=-1,
                 param_grid={'max_depth': [2, 3, 4, 6],
                             'max_features': ['log2', 'sqrt'],
                             'min_samples_leaf': [0.12, 0.14, 0.16, 0.18, 0.25],
                             'n_estimators': [300, 400, 500, 800]},
                 pre_dispatch='2*n_jobs', refit=True, return_train_score=False,
                 scoring='neg_mean_squared_error', verbose=0)




```
# Extract the best estimator
best_model = grid_rf.best_estimator_
best_hyperparams = grid_rf.best_params_
print('Best hyerparameters:\n', best_hyperparams)

# Predict the test set probabilities of the positive class
y_pred_proba = best_model.predict_proba(X_test)[:,1]

# Compute test_roc_auc
test_roc_auc = roc_auc_score(y_test, y_pred_proba)

# Print test_roc_auc
print('Test set ROC AUC score: {:.3f}'.format(test_roc_auc))
```

    Best hyerparameters:
     {'max_depth': 2, 'max_features': 'log2', 'min_samples_leaf': 0.12, 'n_estimators': 300}
    Test set ROC AUC score: 0.678


## Feature Importance##


```
plt.rcParams["figure.figsize"] = (10,10)
importance_rf=pd.Series(best_model.feature_importances_, index=X.columns)

#Sort importance_rf

sorted_importance_rf= importance_rf.sort_values()
sorted_importance_rf.plot(kind='barh');
```

Contrast results RF- DC


```
def cv_roc_tree(X, y, n_split, algo='Tree'):
    
    cv = StratifiedKFold(n_split)
    
    tprs = []
    aucs = []
    mean_fpr = np.linspace(0, 1, 100)
    for train, test in cv.split(X, y):
        if algo == 'Tree':
            clf = DecisionTreeClassifier()
        else:
            clf = RandomForestClassifier(n_estimators=100, max_depth=3, random_state=0)
                
        clf = clf.fit(X.iloc[train], y.iloc[train])
        probas_ = clf.predict_proba(X.iloc[test])
        
        # Compute ROC curve and area the curve
        fpr, tpr, thresholds = roc_curve(y.iloc[test], probas_[:,1])
        tprs.append(np.interp(mean_fpr, fpr, tpr))
        tprs[-1][0] = 0.0
        roc_auc = auc(fpr, tpr)
        aucs.append(roc_auc)
    
    return tprs, aucs, mean_fpr
```


```
tprs_t, aucs_t, mean_fpr_t = cv_roc_tree(df_pre, df['loan_status'].cat.codes, 5)
tprs_f, aucs_f, mean_fpr_f = cv_roc_tree(df_pre, df['loan_status'].cat.codes,5, algo='forest')

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='blue', label='Random guess', alpha=.8)

mean_tpr_t = np.mean(tprs_t, axis=0)
mean_tpr_f = np.mean(tprs_f, axis=0)
mean_tpr_t[-1] = 1.0
mean_tpr_f[-1] = 1.0

mean_auc_t = auc(mean_fpr_t, mean_tpr_t)
mean_auc_f = auc(mean_fpr_f, mean_tpr_f)
std_auc_t = np.std(aucs_t)
std_auc_f = np.std(aucs_f)

plt.plot(mean_fpr_t, mean_tpr_t, color='darkorange', lw=2, alpha=.8,
         label=f"Decision Tree: Cross-validated Mean ROC (AUC = {mean_auc_t:0.2f} $\pm$ {std_auc_t:0.2f})")
plt.plot(mean_fpr_f, mean_tpr_f, color='red', lw=2, alpha=.8,
         label=f"Random Forest: Cross-validated Mean ROC (AUC = {mean_auc_f:0.2f} $\pm$ {std_auc_f:0.2f})")

plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend(loc='lower right')
plt.title('Tree-based Classification of Defaulters: 5-fold Cross-validation')
plt.show()
```


```

```

33076db96e5cf84eb892a58abf1e1fba3336ab43
ipynb
Jupyter Notebook
Model_LOAN_Random_Forest.ipynb
ancastillar/Study-on-the-probability-of-payment-of-a-credit-requested-by-a-client
4bbda2a5ad1ddae38bf444231384d830805a9e1c
CNRI-Python
Model_LOAN_Random_Forest.ipynb
ancastillar/Study-on-the-probability-of-payment-of-a-credit-requested-by-a-client
4bbda2a5ad1ddae38bf444231384d830805a9e1c
CNRI-Python
Model_LOAN_Random_Forest.ipynb
ancastillar/Study-on-the-probability-of-payment-of-a-credit-requested-by-a-client
4bbda2a5ad1ddae38bf444231384d830805a9e1c
CNRI-Python
Qwen/Qwen-72B
1. YES
2. YES