<reponame>mikema2019/Machine-learning-algorithms<filename>Supervised Machine Learning Algorithms/Ensemble Methods/Gradient_Boosting.py

from Decision_Tree_CART import DecisionTree_CART
import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import GradientBoostingRegressor,AdaBoostRegressor

def L(y_train,f):
    return np.sum((y_train-f)**2)
    
def der_L(y_train,f):
    return 2*(y_train-f)
    
def opt_L(y_train,f=0):
    return np.average(y_train-f)

class GradientBoosting():
    def __init__(self,epsilon=[10**-2,10**-2,10**-2],M=3):
        self.func_no=M
        self.model=[]
        self.epsilon=epsilon

    def train(self,X_train,y_train,X_dev,y_dev):
        train_no,feature_no=np.shape(X_train)
        r=np.zeros((train_no,self.func_no+1))
        f_func=[]
        self.f0=opt_L(y_train)
        f=opt_L(y_train)
        r[:,0]=der_L(y_train,f)
        for m in range(self.func_no):
            model=DecisionTree_CART(epsilon=self.epsilon[m])
            node=model.train(X_train,r[:,m],X_dev,y_dev)

            for key,value in node.items():
                if value[7]==True:
                    X_train_temp,y_train_temp,father_no,C,mini_feature_no,length,s_point,leaf_node=value
                    f_temp=0
                    for i in range(m):
                        f_temp+=f_func[i].predict(X_train_temp)
                    f_temp+=self.f0
                    index=[]
                    for X in X_train_temp:
                        index.append(list(X_train).index(X))                       
                    C=opt_L(y_train[index],f_temp)
                    node[key]=X_train_temp,y_train_temp,father_no,C,mini_feature_no,length,s_point,leaf_node

            f_func.append(model)
            f=f+model.predict(X_train)
            r[:,m+1]=der_L(y_train,f)              
            self.model.append(model)
        
    def predict(self,X_test):
        y_test=0
        for model in self.model:
            y_test+=model.predict(X_test)
        return y_test+self.f0

if __name__=='__main__':    
#    np.random.seed(4)    
    X_train=10*np.random.rand(300,1)
    y_train=np.sin(X_train)
    y_train=y_train.ravel()
    X_dev=10*np.random.rand(100,1)
    y_dev=np.sin(X_dev)    
    X_test_unsorted=10*np.random.rand(100,1)
#    X_test_unsorted=X_train            
    X_test=np.sort(X_test_unsorted,axis=0)
    y_test=np.sin(X_test)

    model=GradientBoosting(epsilon=[10**-8,10**-8,10**-8])
    model.train(X_train,y_train,X_dev,y_dev)
    y_pred=model.predict(X_test)
#    print('Decision Tree', y_pred)
    print('Gradient_Boosting:',np.average((y_test-y_pred)**2))
    plt.figure(1)
    plt.plot(X_test,y_pred,label='y_pred')
    plt.scatter(X_test,y_test,label='y_test')
#    plt.scatter(X_train,y_train,label='y_train')
    plt.legend() 

    
    model=DecisionTree_CART(epsilon=10**-8,method='Regression')
    model.train(X_train,y_train,X_dev,y_dev)
    y_pred=model.predict(X_test)
#    print('Decision Tree', y_pred)
    print('DecisionTree_CART:',np.average((y_test-y_pred)**2))

    
    model = AdaBoostRegressor()
    model.fit(X_train, y_train)
    y_pred=model.predict(X_test)
#    print('sklearn', y_pred)
    print('sklearn_AB:',np.average((y_test-y_pred)**2))
    
    model = GradientBoostingRegressor()
    model.fit(X_train, y_train)
    y_pred=model.predict(X_test)
#    print('sklearn', y_pred)
    print('sklearn_GBR:',np.average((y_test-y_pred)**2))    
        
    plt.figure(2)
    plt.plot(X_test,y_pred,label='y_pred')
    plt.scatter(X_test,y_test,label='y_test')
    plt.legend()
            

            
        
    
Machine Learning Algorithms/Ensemble Methods/Gradient_Boosting.py

from Decision_Tree_CART import DecisionTree_CART
import numpy as np
import matplotlib.pyplot as plt
from sklearn.ensemble import GradientBoostingRegressor,AdaBoostRegressor

def L(y_train,f):
    return np.sum((y_train-f)**2)
    
def der_L(y_train,f):
    return 2*(y_train-f)
    
def opt_L(y_train,f=0):
    return np.average(y_train-f)

class GradientBoosting():
    def __init__(self,epsilon=[10**-2,10**-2,10**-2],M=3):
        self.func_no=M
        self.model=[]
        self.epsilon=epsilon

    def train(self,X_train,y_train,X_dev,y_dev):
        train_no,feature_no=np.shape(X_train)
        r=np.zeros((train_no,self.func_no+1))
        f_func=[]
        self.f0=opt_L(y_train)
        f=opt_L(y_train)
        r[:,0]=der_L(y_train,f)
        for m in range(self.func_no):
            model=DecisionTree_CART(epsilon=self.epsilon[m])
            node=model.train(X_train,r[:,m],X_dev,y_dev)

            for key,value in node.items():
                if value[7]==True:
                    X_train_temp,y_train_temp,father_no,C,mini_feature_no,length,s_point,leaf_node=value
                    f_temp=0
                    for i in range(m):
                        f_temp+=f_func[i].predict(X_train_temp)
                    f_temp+=self.f0
                    index=[]
                    for X in X_train_temp:
                        index.append(list(X_train).index(X))                       
                    C=opt_L(y_train[index],f_temp)
                    node[key]=X_train_temp,y_train_temp,father_no,C,mini_feature_no,length,s_point,leaf_node

            f_func.append(model)
            f=f+model.predict(X_train)
            r[:,m+1]=der_L(y_train,f)              
            self.model.append(model)
        
    def predict(self,X_test):
        y_test=0
        for model in self.model:
            y_test+=model.predict(X_test)
        return y_test+self.f0

if __name__=='__main__':    
#    np.random.seed(4)    
    X_train=10*np.random.rand(300,1)
    y_train=np.sin(X_train)
    y_train=y_train.ravel()
    X_dev=10*np.random.rand(100,1)
    y_dev=np.sin(X_dev)    
    X_test_unsorted=10*np.random.rand(100,1)
#    X_test_unsorted=X_train            
    X_test=np.sort(X_test_unsorted,axis=0)
    y_test=np.sin(X_test)

    model=GradientBoosting(epsilon=[10**-8,10**-8,10**-8])
    model.train(X_train,y_train,X_dev,y_dev)
    y_pred=model.predict(X_test)
#    print('Decision Tree', y_pred)
    print('Gradient_Boosting:',np.average((y_test-y_pred)**2))
    plt.figure(1)
    plt.plot(X_test,y_pred,label='y_pred')
    plt.scatter(X_test,y_test,label='y_test')
#    plt.scatter(X_train,y_train,label='y_train')
    plt.legend() 

    
    model=DecisionTree_CART(epsilon=10**-8,method='Regression')
    model.train(X_train,y_train,X_dev,y_dev)
    y_pred=model.predict(X_test)
#    print('Decision Tree', y_pred)
    print('DecisionTree_CART:',np.average((y_test-y_pred)**2))

    
    model = AdaBoostRegressor()
    model.fit(X_train, y_train)
    y_pred=model.predict(X_test)
#    print('sklearn', y_pred)
    print('sklearn_AB:',np.average((y_test-y_pred)**2))
    
    model = GradientBoostingRegressor()
    model.fit(X_train, y_train)
    y_pred=model.predict(X_test)
#    print('sklearn', y_pred)
    print('sklearn_GBR:',np.average((y_test-y_pred)**2))    
        
    plt.figure(2)
    plt.plot(X_test,y_pred,label='y_pred')
    plt.scatter(X_test,y_test,label='y_test')
    plt.legend()
#    np.random.seed(4) #    X_test_unsorted=X_train #    print('Decision Tree', y_pred) #    plt.scatter(X_train,y_train,label='y_train') #    print('Decision Tree', y_pred) #    print('sklearn', y_pred) #    print('sklearn', y_pred)