<filename>final_code.py<gh_stars>0
# -*- coding: utf-8 -*-
"""
Created on Sat May  5 19:11:51 2018

@author: brahe
"""

# =============================================================================
# PlEASE MAKE SURE THAT TENSOR FLOW AND KERAS 2.1.3 IS INSTALLED 
# IF NOT USE THE COMMAND: pip3 install --upgrade keras==2.1.3
# We are using: Python 3.5.4 |Anaconda, Inc.| (default, Nov  8 2017, 14:34:30) [MSC v.1900 64 bit (AMD64)]
# IPython 6.2.1 -- An enhanced Interactive Python.
# =============================================================================

import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.metrics import confusion_matrix
import itertools
import matplotlib.pyplot as plt
# Importing the Keras libraries 

from keras.layers import Dropout
from keras.layers.core import  Activation
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.models import load_model
from keras.models import model_from_json

#Set the working directory to the folder data 

# =============================================================================
#                 Part 1: Building model      
# =============================================================================


# Importing the dataset
def read_data():
    y = pd.read_csv('Classes1000.csv' , sep = ';')
    X = pd.read_csv('portf1000.csv' , sep = ';')
    X = X.iloc[:, 1:].values
    y = y.iloc[:, 2].values
    y = y-1
    return X, y




# Tranforming y to the same length of X
def data_same_length(X, y, nb_of_portfolios, size_of_portfolio):     
    y_portfolios = []
    for i in range(nb_of_portfolios):
        for j in range(size_of_portfolio):
            y_portfolios.append(y[i,])
    return y_portfolios


 
# Split data into train and test       
def split_train_test(grou, train_size,
                     portfolio_size,
                     nb_of_features,
                     nb_of_portfolios):
    
    X_train=[]
    X_test=[]
    y_train=[]
    y_test=[]
    
    for k in range(nb_of_portfolios):
        if k < int(train_size * nb_of_portfolios):
            X_train.append(grou[k, 0:portfolio_size, 0:nb_of_features])
            y_train.append(grou[k, 0:1, nb_of_features:nb_of_features+1])
        else:
            X_test.append(grou[k, 0:portfolio_size, 0:nb_of_features])
            y_test.append(grou[k, 0:1, nb_of_features:nb_of_features+1])
            
    X_test = np.array(X_test)
    X_train = np.array(X_train)
    y_train = np.array(y_train)
    y_test = np.array(y_test)
    
    return X_test, X_train, y_train, y_test


# Encoding categorical data
def transform_categorical_variables(data):
    labelencoder_X_1 = LabelEncoder()
    data = labelencoder_X_1.fit_transform(data)
    labelencoder_X_2 = LabelEncoder()
    
    data = data.reshape(len(data),1)
    onehotencoder = OneHotEncoder(categorical_features = [0])
    data = onehotencoder.fit_transform(data).toarray()
    return data






# Convolutional NN architecture

def convolutional_NN_model(X_train, X_test, y_train,
                           y_test,
                           batch_size,
                           nb_epoch):
    
    # Step1 Convolutional layer
    classifier = Sequential()
    classifier.add(Conv2D(32, (1, 3), input_shape = (1, 20, 38),
                          activation = 'tanh'))
    
    # Step 2 - Pooling
    
    classifier.add(MaxPooling2D(pool_size = (1, 3)))
    
    # Step 2(b) - Add 2nd Convolution Layer making it Deep followed by a Pooling Layer
    
    classifier.add(Conv2D(32, (1, 3), activation = 'tanh'))
    classifier.add(MaxPooling2D(pool_size = (1, 2)))
    
    # Step 3 - Flattening
    
    classifier.add(Flatten())
    
    # Step 4 - Fully Connected Neural Network
    
    # Hidden Layer - Activation Function RELU
    classifier.add(Dense(units = 256, activation = 'tanh')) 
    # Output Layer - Activation Function Softmax(to clasify classes)
    classifier.add(Dense(units = 2, activation = 'softmax'))
    
    # Compile the CNN
    
    # Binary Crossentropy - to classify between good and bad portfolios
    classifier.compile(optimizer = 'rmsprop', loss = 'binary_crossentropy', 
                       metrics = ['accuracy'])
    
    classifier.fit(X_train, y_train,batch_size, nb_epoch,
                   validation_data=(X_test, y_test))
    
    return classifier


# transforming 2d array into a list

def transform_matrix_tolist(y_test):
    y_test_array = []
    for f in range(len(y_test)):
        if y_test[f][0] == 1:
            y_test_array.append(0) 
        elif y_test[f][1] == 1:
            y_test_array.append(1)
            
    y_test_array = np.array(y_test_array) 
    return y_test_array






if __name__ == "__main__":
    
    # Importing the dataset
    X, y = read_data()
    
    # Tranforming y to the same length of X
    y_portfolios = data_same_length(X, y, nb_of_portfolios = 1000,
                                    size_of_portfolio = 20)
     
    # missing data 
    X = pd.DataFrame(X)    
    X = X.fillna(0)
    
    # Concatenate X and y
    y = pd.DataFrame(y)
    y = np.array(y)
    y_portfolios = np.array(y_portfolios) 
    y_portfolios = pd.DataFrame(y_portfolios) 
    grou = pd.concat((X,y_portfolios),axis=1)
    grou = np.array(grou)
    
    # Reshape grou into 3 dimensions tensor
    grou = grou.reshape(y.shape[0], 20, 39)
  
    # Split data into train and test
    X_test, X_train, y_train, y_test = split_train_test(grou,
                                                        train_size = 0.8,
                                                        portfolio_size = 20,
                                                        nb_of_features = 38,
                                                        nb_of_portfolios = 1000)
    


    # Reshape y
    y_test = y_test.reshape(200,1)
    y_train = y_train.reshape(800,1)

    # Encoding categorical data
    y_train = transform_categorical_variables(data = y_train)
    y_test = transform_categorical_variables(y_test)
    
    # Reshape into 4d tensors
    X_train = np.reshape(X_train, (800, 1, 20, 38))
    X_test = np.reshape(X_test, (200, 1, 20, 38))
    
        #fit model
    classifier = convolutional_NN_model(X_train, X_test, y_train, y_test, 1, 100)
    
    
# =============================================================================
#                     Part 2: Model evaluation
# =============================================================================
    
    # Prediction
    y_pred = classifier.predict_classes(X_test, batch_size = 1)
    
    # Transforming 2d array into a list
    y_test_array = transform_matrix_tolist(y_test) 
        # Confusion matrix
    confusion_matrix(y_pred, y_test_array)
    
    # Classification report
    from sklearn.metrics import classification_report
    target_names = ['good', 'bad']
    print(classification_report(y_pred, y_test_array, target_names=target_names))

    
    
    
# =============================================================================
#                   Part 3:   Model summary
# =============================================================================

# Model summary (architectures)
classifier.summary()

# model Configurations
classifier.get_config()

# Number of parameters in the model
classifier.count_params()

# Model weights
classifier.get_weights()

      
# =============================================================================
#              Part 4:  Save trained model parameters         
# =============================================================================

## serialize weights to HDF5
#classifier.save_weights("pre_trained/final_model.h5")

#print("Saved model to disk")
#model_json = classifier.to_json()
#with open("pre_trained/final_model", "w") as json_file:
#    json_file.write(model_json)

# =============================================================================
#                 Part5: Load Pre-trained model
# =============================================================================
 
# load json and create model
json_file = open('pre_trained/final_model', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)

# load weights into new model
loaded_model.load_weights("pre_trained/final_model.h5")
print("Loaded model from disk")
loaded_model.compile(loss='binary_crossentropy',
                     optimizer='rmsprop',
                     metrics=['accuracy'])

loaded_model.fit(X_train, y_train,batch_size=1, nb_epoch=1,
                 validation_data=(X_test, y_test))

# evaluate loaded model on test data
score = loaded_model.evaluate(X_test, y_test, verbose=0)
print(score)



# =============================================================================
#                  Part 6:Confusion matrix
# =============================================================================

       




def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test_array, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes= ("good", "bad"),
                      title='Confusion matrix, without normalization')

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=("good" "bad"), normalize=True,
                      title='Normalized confusion matrix')

plt.show()




<filename>final_code.py<gh_stars>0
# -*- coding: utf-8 -*-
"""
Created on Sat May  5 19:11:51 2018

@author: brahe
"""

# =============================================================================
# PlEASE MAKE SURE THAT TENSOR FLOW AND KERAS 2.1.3 IS INSTALLED 
# IF NOT USE THE COMMAND: pip3 install --upgrade keras==2.1.3
# We are using: Python 3.5.4 |Anaconda, Inc.| (default, Nov  8 2017, 14:34:30) [MSC v.1900 64 bit (AMD64)]
# IPython 6.2.1 -- An enhanced Interactive Python.
# =============================================================================

import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.metrics import confusion_matrix
import itertools
import matplotlib.pyplot as plt
# Importing the Keras libraries 

from keras.layers import Dropout
from keras.layers.core import  Activation
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.models import load_model
from keras.models import model_from_json

#Set the working directory to the folder data 

# =============================================================================
#                 Part 1: Building model      
# =============================================================================


# Importing the dataset
def read_data():
    y = pd.read_csv('Classes1000.csv' , sep = ';')
    X = pd.read_csv('portf1000.csv' , sep = ';')
    X = X.iloc[:, 1:].values
    y = y.iloc[:, 2].values
    y = y-1
    return X, y




# Tranforming y to the same length of X
def data_same_length(X, y, nb_of_portfolios, size_of_portfolio):     
    y_portfolios = []
    for i in range(nb_of_portfolios):
        for j in range(size_of_portfolio):
            y_portfolios.append(y[i,])
    return y_portfolios


 
# Split data into train and test       
def split_train_test(grou, train_size,
                     portfolio_size,
                     nb_of_features,
                     nb_of_portfolios):
    
    X_train=[]
    X_test=[]
    y_train=[]
    y_test=[]
    
    for k in range(nb_of_portfolios):
        if k < int(train_size * nb_of_portfolios):
            X_train.append(grou[k, 0:portfolio_size, 0:nb_of_features])
            y_train.append(grou[k, 0:1, nb_of_features:nb_of_features+1])
        else:
            X_test.append(grou[k, 0:portfolio_size, 0:nb_of_features])
            y_test.append(grou[k, 0:1, nb_of_features:nb_of_features+1])
            
    X_test = np.array(X_test)
    X_train = np.array(X_train)
    y_train = np.array(y_train)
    y_test = np.array(y_test)
    
    return X_test, X_train, y_train, y_test


# Encoding categorical data
def transform_categorical_variables(data):
    labelencoder_X_1 = LabelEncoder()
    data = labelencoder_X_1.fit_transform(data)
    labelencoder_X_2 = LabelEncoder()
    
    data = data.reshape(len(data),1)
    onehotencoder = OneHotEncoder(categorical_features = [0])
    data = onehotencoder.fit_transform(data).toarray()
    return data






# Convolutional NN architecture

def convolutional_NN_model(X_train, X_test, y_train,
                           y_test,
                           batch_size,
                           nb_epoch):
    
    # Step1 Convolutional layer
    classifier = Sequential()
    classifier.add(Conv2D(32, (1, 3), input_shape = (1, 20, 38),
                          activation = 'tanh'))
    
    # Step 2 - Pooling
    
    classifier.add(MaxPooling2D(pool_size = (1, 3)))
    
    # Step 2(b) - Add 2nd Convolution Layer making it Deep followed by a Pooling Layer
    
    classifier.add(Conv2D(32, (1, 3), activation = 'tanh'))
    classifier.add(MaxPooling2D(pool_size = (1, 2)))
    
    # Step 3 - Flattening
    
    classifier.add(Flatten())
    
    # Step 4 - Fully Connected Neural Network
    
    # Hidden Layer - Activation Function RELU
    classifier.add(Dense(units = 256, activation = 'tanh')) 
    # Output Layer - Activation Function Softmax(to clasify classes)
    classifier.add(Dense(units = 2, activation = 'softmax'))
    
    # Compile the CNN
    
    # Binary Crossentropy - to classify between good and bad portfolios
    classifier.compile(optimizer = 'rmsprop', loss = 'binary_crossentropy', 
                       metrics = ['accuracy'])
    
    classifier.fit(X_train, y_train,batch_size, nb_epoch,
                   validation_data=(X_test, y_test))
    
    return classifier


# transforming 2d array into a list

def transform_matrix_tolist(y_test):
    y_test_array = []
    for f in range(len(y_test)):
        if y_test[f][0] == 1:
            y_test_array.append(0) 
        elif y_test[f][1] == 1:
            y_test_array.append(1)
            
    y_test_array = np.array(y_test_array) 
    return y_test_array






if __name__ == "__main__":
    
    # Importing the dataset
    X, y = read_data()
    
    # Tranforming y to the same length of X
    y_portfolios = data_same_length(X, y, nb_of_portfolios = 1000,
                                    size_of_portfolio = 20)
     
    # missing data 
    X = pd.DataFrame(X)    
    X = X.fillna(0)
    
    # Concatenate X and y
    y = pd.DataFrame(y)
    y = np.array(y)
    y_portfolios = np.array(y_portfolios) 
    y_portfolios = pd.DataFrame(y_portfolios) 
    grou = pd.concat((X,y_portfolios),axis=1)
    grou = np.array(grou)
    
    # Reshape grou into 3 dimensions tensor
    grou = grou.reshape(y.shape[0], 20, 39)
  
    # Split data into train and test
    X_test, X_train, y_train, y_test = split_train_test(grou,
                                                        train_size = 0.8,
                                                        portfolio_size = 20,
                                                        nb_of_features = 38,
                                                        nb_of_portfolios = 1000)
    


    # Reshape y
    y_test = y_test.reshape(200,1)
    y_train = y_train.reshape(800,1)

    # Encoding categorical data
    y_train = transform_categorical_variables(data = y_train)
    y_test = transform_categorical_variables(y_test)
    
    # Reshape into 4d tensors
    X_train = np.reshape(X_train, (800, 1, 20, 38))
    X_test = np.reshape(X_test, (200, 1, 20, 38))
    
        #fit model
    classifier = convolutional_NN_model(X_train, X_test, y_train, y_test, 1, 100)
    
    
# =============================================================================
#                     Part 2: Model evaluation
# =============================================================================
    
    # Prediction
    y_pred = classifier.predict_classes(X_test, batch_size = 1)
    
    # Transforming 2d array into a list
    y_test_array = transform_matrix_tolist(y_test) 
        # Confusion matrix
    confusion_matrix(y_pred, y_test_array)
    
    # Classification report
    from sklearn.metrics import classification_report
    target_names = ['good', 'bad']
    print(classification_report(y_pred, y_test_array, target_names=target_names))

    
    
    
# =============================================================================
#                   Part 3:   Model summary
# =============================================================================

# Model summary (architectures)
classifier.summary()

# model Configurations
classifier.get_config()

# Number of parameters in the model
classifier.count_params()

# Model weights
classifier.get_weights()

      
# =============================================================================
#              Part 4:  Save trained model parameters         
# =============================================================================

## serialize weights to HDF5
#classifier.save_weights("pre_trained/final_model.h5")

#print("Saved model to disk")
#model_json = classifier.to_json()
#with open("pre_trained/final_model", "w") as json_file:
#    json_file.write(model_json)

# =============================================================================
#                 Part5: Load Pre-trained model
# =============================================================================
 
# load json and create model
json_file = open('pre_trained/final_model', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)

# load weights into new model
loaded_model.load_weights("pre_trained/final_model.h5")
print("Loaded model from disk")
loaded_model.compile(loss='binary_crossentropy',
                     optimizer='rmsprop',
                     metrics=['accuracy'])

loaded_model.fit(X_train, y_train,batch_size=1, nb_epoch=1,
                 validation_data=(X_test, y_test))

# evaluate loaded model on test data
score = loaded_model.evaluate(X_test, y_test, verbose=0)
print(score)



# =============================================================================
#                  Part 6:Confusion matrix
# =============================================================================

       




def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test_array, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes= ("good", "bad"),
                      title='Confusion matrix, without normalization')

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=("good" "bad"), normalize=True,
                      title='Normalized confusion matrix')

plt.show()




# -*- coding: utf-8 -*- Created on Sat May  5 19:11:51 2018  @author: brahe # ============================================================================= # PlEASE MAKE SURE THAT TENSOR FLOW AND KERAS 2.1.3 IS INSTALLED # IF NOT USE THE COMMAND: pip3 install --upgrade keras==2.1.3 # We are using: Python 3.5.4 |Anaconda, Inc.| (default, Nov  8 2017, 14:34:30) [MSC v.1900 64 bit (AMD64)] # IPython 6.2.1 -- An enhanced Interactive Python. # ============================================================================= # Importing the Keras libraries #Set the working directory to the folder data # ============================================================================= #                 Part 1: Building model # ============================================================================= # Importing the dataset # Tranforming y to the same length of X # Split data into train and test # Encoding categorical data # Convolutional NN architecture # Step1 Convolutional layer # Step 2 - Pooling # Step 2(b) - Add 2nd Convolution Layer making it Deep followed by a Pooling Layer # Step 3 - Flattening # Step 4 - Fully Connected Neural Network # Hidden Layer - Activation Function RELU # Output Layer - Activation Function Softmax(to clasify classes) # Compile the CNN # Binary Crossentropy - to classify between good and bad portfolios # transforming 2d array into a list # Importing the dataset # Tranforming y to the same length of X # missing data # Concatenate X and y # Reshape grou into 3 dimensions tensor # Split data into train and test # Reshape y # Encoding categorical data # Reshape into 4d tensors #fit model # ============================================================================= #                     Part 2: Model evaluation # ============================================================================= # Prediction # Transforming 2d array into a list # Confusion matrix # Classification report # ============================================================================= #                   Part 3:   Model summary # ============================================================================= # Model summary (architectures) # model Configurations # Number of parameters in the model # Model weights # ============================================================================= #              Part 4:  Save trained model parameters # ============================================================================= ## serialize weights to HDF5 #classifier.save_weights("pre_trained/final_model.h5") #print("Saved model to disk") #model_json = classifier.to_json() #with open("pre_trained/final_model", "w") as json_file: #    json_file.write(model_json) # ============================================================================= #                 Part5: Load Pre-trained model # ============================================================================= # load json and create model # load weights into new model # evaluate loaded model on test data # ============================================================================= #                  Part 6:Confusion matrix # ============================================================================= This function prints and plots the confusion matrix.     Normalization can be applied by setting `normalize=True`. # Compute confusion matrix # Plot non-normalized confusion matrix # Plot normalized confusion matrix