<reponame>tala360/CIFAR10-CNN
# -*- coding: utf-8 -*-
"""Neural Networks Final.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/10Lsy5WmktihXzjDCz2eNCoXLHk4o0X1v

# Neural Networks Coursework Assignment
## Multi-class Classification: CIFAR-10 Dataset
### Candidate No. 215921 
### Dr. <NAME>

# Data Loading and Visualisation

1. Import all necessary libraries, including the dataset
"""

from matplotlib import pyplot as plt
import numpy as np
import tensorflow as tf
import cv2
import keras
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping
from tensorflow.keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout,Activation,BatchNormalization
from keras.callbacks import ModelCheckpoint, History
from keras.optimizers import Adam, Adadelta, SGD, RMSprop
from keras.models import load_model
from keras.utils.vis_utils import plot_model
from keras import regularizers
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from keras.regularizers import l2,l1
# Dataset library
from keras.datasets import cifar10

"""2. Load the dataset as a tuple of numpy arrays"""

(x_train,y_train),(x_test,y_test) = cifar10.load_data()

# 50,000 training and 10,000 test, (32x32x3 for RGB images, 1 for labels being an integer 0-9)
# each label denotes one of 10 classes respectively (0-9) => airplane, automobile, bird, cat, deer, dog, frog,horse, ship, truck
print('Training images shape:',x_train.shape)
print('Training labels shape:',y_train.shape)
print('Test images shape:',x_test.shape)
print('Test labels shape:',y_test.shape)

# Random data visualisation function that prints the labels as well.
def visualise_data(data,y_data):
    f, axarr = plt.subplots(2,2)

    img1 = np.random.randint(0, data.shape[0])
    axarr[0,0].imshow(data[img1])

    img2 = np.random.randint(0, data.shape[0])
    axarr[0,1].imshow(data[img2])

    img3 = np.random.randint(0, data.shape[0])
    axarr[1,0].imshow(data[img3])

    img4 = np.random.randint(0, data.shape[0])
    axarr[1,1].imshow(data[img4])

    # show the label of each image above it
    label1 = y_data[img1][0]
    axarr[0,0].title.set_text(label1)
    label2 = y_data[img2][0]
    axarr[0,1].title.set_text(label2)
    label3 = y_data[img3][0]
    axarr[1,0].title.set_text(label3)
    label4 = y_data[img4][0]
    axarr[1,1].title.set_text(label4)

    plt.tight_layout()

# Visualise the training data
visualise_data(x_train,y_train)

"""# Data Preprocessing

3. Convert the image data to 32-bit precision and normalise it between 0-1
"""

x_train_norm = x_train.astype('float32')/255.0
x_test_norm = x_test.astype('float32')/255.0

"""4. Convert the image data to grayscale"""

xtrain_gray = []
for i in range(0,x_train_norm.shape[0]):
  xtrain_gray.append(cv2.cvtColor(x_train_norm[i],cv2.COLOR_BGR2GRAY))
xtrain_gray = np.array(xtrain_gray)
xtrain_gray.shape

xtest_gray = []
for i in range(0,x_test_norm.shape[0]):
  xtest_gray.append(cv2.cvtColor(x_test_norm[i],cv2.COLOR_BGR2GRAY))
xtest_gray = np.array(xtest_gray)
xtest_gray.shape

plt.imshow(xtrain_gray[0],cmap='gray')

"""5. Apply one-hot encoding for the label data """

enc_ytrain = to_categorical(y_train)
enc_ytest = to_categorical(y_test)

y_train[0]

"""It can be seen that the first element of ytrain has a label of 6. Thus, it is categorised at the 6th index as seen below."""

enc_ytrain[0]

"""# Building the model"""

def plot_acc_loss(hist,data_title=''):
    # loss plot 
    plt.figure(figsize=(16,5))
    plt.subplot(1, 2, 1)
    plt.suptitle(data_title, fontsize=10)
    plt.ylabel('Loss', fontsize=16)
    plt.xlabel('Epochs', fontsize=16)
    plt.plot(hist.history['loss'], color='g', label='Training Loss')
    plt.plot(hist.history['val_loss'], color='r', label='Validation Loss')
    plt.legend(loc='upper right')

    # accuracy plot
    plt.subplot(1, 2, 2)
    plt.ylabel('Accuracy', fontsize=16)
    plt.xlabel('Epochs', fontsize=16)
    plt.plot(hist.history['accuracy'], color='g', label='Training Accuracy')
    plt.plot(hist.history['val_accuracy'], color='r', label='Validation Accuracy')
    plt.legend(loc='lower right')
    plt.show()

"""#### 1. Model without Batch Normalisation"""

hist = History()

input_shape = (32,32,1)
epochs = 200
batch_size = 128
classes = 10
xtrain_ = xtrain_gray.reshape(50000,32,32,1)

model1 = Sequential()

model1.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
#model1.add(BatchNormalization())
model1.add(MaxPooling2D(pool_size=2))
model1.add(Dropout(0.3))

model1.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
#model1.add(BatchNormalization())
model1.add(MaxPooling2D(pool_size=2))
model1.add(Dropout(0.4))

model1.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
#model1.add(BatchNormalization())
model1.add(MaxPooling2D(pool_size=2))
model1.add(Dropout(0.5))

model1.add(Flatten())
model1.add(Dense(classes, activation='softmax'))
print(model1.summary())

model1.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist = model1.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist,'Original CIFAR-10 Data (Grayscale, Normalised) [Without Batch Normalisation]')

hist2 = History()

model2 = Sequential()

model2.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=2))
model2.add(Dropout(0.3))

model2.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=2))
model2.add(Dropout(0.4))

model2.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=2))
model2.add(Dropout(0.5))

model2.add(Flatten())
model2.add(Dense(classes, activation='softmax'))
print(model2.summary())

model2.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist2 = model2.fit(xtrain_, enc_ytrain,batch_size=batch_size,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist2,'Original CIFAR-10 Data (Grayscale, Normalised) [With Batch Normalisation]')

X_train, X_val, y_train, y_val = train_test_split(xtrain_, enc_ytrain,shuffle=True)

hist3 = History()

model3 = Sequential()

model3.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model3.add(BatchNormalization())
model3.add(MaxPooling2D(pool_size=2))
model3.add(Dropout(0.3))

model3.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model3.add(BatchNormalization())
model3.add(MaxPooling2D(pool_size=2))
model3.add(Dropout(0.4))

model3.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model3.add(BatchNormalization())
model3.add(MaxPooling2D(pool_size=2))
model3.add(Dropout(0.5))

model3.add(Flatten())
model3.add(Dense(classes, activation='softmax'))
print(model3.summary())

model3.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist3 = model3.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist3,'Original CIFAR-10 Data (Grayscale, Normalised) [Learning Rate = 1e-3]')

hist4 = History()

model4 = Sequential()

model4.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model4.add(BatchNormalization())
model4.add(MaxPooling2D(pool_size=2))
model4.add(Dropout(0.3))

model4.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model4.add(BatchNormalization())
model4.add(MaxPooling2D(pool_size=2))
model4.add(Dropout(0.4))

model4.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model4.add(BatchNormalization())
model4.add(MaxPooling2D(pool_size=2))
model4.add(Dropout(0.5))

model4.add(Flatten())
model4.add(Dense(classes, activation='softmax'))
print(model4.summary())

model4.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-5), metrics=['accuracy','CategoricalAccuracy'])

hist4 = model4.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist4,'Original CIFAR-10 Data (Grayscale, Normalised) [Learning Rate = 1e-5]')

hist5 = History()
model5 = Sequential()

model5.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model5.add(BatchNormalization())
model5.add(MaxPooling2D(pool_size=2))
model5.add(Dropout(0.3))

model5.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model5.add(BatchNormalization())
model5.add(MaxPooling2D(pool_size=2))
model5.add(Dropout(0.4))

model5.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model5.add(BatchNormalization())
model5.add(MaxPooling2D(pool_size=2))
model5.add(Dropout(0.5))

model5.add(Flatten())
model5.add(Dense(classes, activation='softmax'))
print(model5.summary())

model5.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-2), metrics=['accuracy','CategoricalAccuracy'])

hist5 = model5.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist5,'Original CIFAR-10 Data (Grayscale, Normalised) [Learning Rate = 1e-2]')

hist6 = History()
model6 = Sequential()

model6.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model6.add(BatchNormalization())
model6.add(MaxPooling2D(pool_size=2))
model6.add(Dropout(0.3))

model6.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model6.add(BatchNormalization())
model6.add(MaxPooling2D(pool_size=2))
model6.add(Dropout(0.4))

model6.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model6.add(BatchNormalization())
model6.add(MaxPooling2D(pool_size=2))
model6.add(Dropout(0.5))

model6.add(Flatten())
model6.add(Dense(classes, activation='softmax'))
print(model6.summary())

model6.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist6 = model6.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist6,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-4]')

hist7 = History()
model7 = Sequential()

model7.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model7.add(BatchNormalization())
model7.add(MaxPooling2D(pool_size=2))
model7.add(Dropout(0.3))

model7.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model7.add(BatchNormalization())
model7.add(MaxPooling2D(pool_size=2))
model7.add(Dropout(0.4))

model7.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model7.add(BatchNormalization())
model7.add(MaxPooling2D(pool_size=2))
model7.add(Dropout(0.5))

model7.add(Flatten())
model7.add(Dense(classes, activation='softmax'))
print(model7.summary())

model7.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-5), metrics=['accuracy','CategoricalAccuracy'])

hist7 = model7.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist7,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-5]')

hist8 = History()
model8 = Sequential()

model8.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model8.add(BatchNormalization())
model8.add(MaxPooling2D(pool_size=2))
model8.add(Dropout(0.3))

model8.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model8.add(BatchNormalization())
model8.add(MaxPooling2D(pool_size=2))
model8.add(Dropout(0.4))

model8.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model8.add(BatchNormalization())
model8.add(MaxPooling2D(pool_size=2))
model8.add(Dropout(0.5))

model8.add(Flatten())
model8.add(Dense(classes, activation='softmax'))
print(model8.summary())

model8.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist8 = model8.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist8,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-3]')

hist9 = History()
model9 = Sequential()

model9.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model9.add(BatchNormalization())
model9.add(MaxPooling2D(pool_size=2))
model9.add(Dropout(0.3))

model9.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model9.add(BatchNormalization())
model9.add(MaxPooling2D(pool_size=2))
model9.add(Dropout(0.4))

model9.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model9.add(BatchNormalization())
model9.add(MaxPooling2D(pool_size=2))
model9.add(Dropout(0.5))

model9.add(Flatten())
model9.add(Dense(classes, activation='softmax'))
print(model9.summary())

model9.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-2), metrics=['accuracy','CategoricalAccuracy'])

hist9 = model9.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist9,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-2]')

hist10 = History()
model10 = Sequential()

model10.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model10.add(BatchNormalization())
model10.add(MaxPooling2D(pool_size=2))
model10.add(Dropout(0.3))

model10.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model10.add(BatchNormalization())
model10.add(MaxPooling2D(pool_size=2))
model10.add(Dropout(0.4))

model10.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model10.add(BatchNormalization())
model10.add(MaxPooling2D(pool_size=2))
model10.add(Dropout(0.5))

model10.add(Flatten())
model10.add(Dense(classes, activation='softmax'))
print(model10.summary())

model10.save('model.h5')

model10.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-2), metrics=['accuracy','CategoricalAccuracy'])

hist10 = model10.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist10,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-2]')

hist11 = History()
model11 = Sequential()

model11.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model11.add(BatchNormalization())
model11.add(MaxPooling2D(pool_size=2))
model11.add(Dropout(0.3))

model11.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model11.add(BatchNormalization())
model11.add(MaxPooling2D(pool_size=2))
model11.add(Dropout(0.4))

model11.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model11.add(BatchNormalization())
model11.add(MaxPooling2D(pool_size=2))
model11.add(Dropout(0.5))

model11.add(Flatten())
model11.add(Dense(classes, activation='softmax'))
print(model11.summary())

model11.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist11 = model11.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist11,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-3]')



hist12 = History()
model12 = Sequential()

model12.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model12.add(BatchNormalization())
model12.add(MaxPooling2D(pool_size=2))
model12.add(Dropout(0.3))

model12.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model12.add(BatchNormalization())
model12.add(MaxPooling2D(pool_size=2))
model12.add(Dropout(0.4))

model12.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model12.add(BatchNormalization())
model12.add(MaxPooling2D(pool_size=2))
model12.add(Dropout(0.5))

model12.add(Flatten())
model12.add(Dense(classes, activation='softmax'))
print(model12.summary())

model12.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist12 = model12.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist12,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-4]')





hist13 = History()
model13 = Sequential()

model13.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model13.add(BatchNormalization())
model13.add(MaxPooling2D(pool_size=2))
model13.add(Dropout(0.3))

model13.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model13.add(BatchNormalization())
model13.add(MaxPooling2D(pool_size=2))
model13.add(Dropout(0.4))

model13.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model13.add(BatchNormalization())
model13.add(MaxPooling2D(pool_size=2))
model13.add(Dropout(0.5))

model13.add(Flatten())
model13.add(Dense(classes, activation='softmax'))
print(model13.summary())

model13.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-5), metrics=['accuracy','CategoricalAccuracy'])

hist13 = model13.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist13,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-5]')



hist14 = History()
model14 = Sequential()

model14.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model14.add(BatchNormalization())
model14.add(MaxPooling2D(pool_size=2))
model14.add(Dropout(0.3))

model14.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model14.add(BatchNormalization())
model14.add(MaxPooling2D(pool_size=2))
model14.add(Dropout(0.4))

# model14.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
# model14.add(BatchNormalization())
# model14.add(MaxPooling2D(pool_size=2))
# model14.add(Dropout(0.5))

model14.add(Flatten())
model14.add(Dense(classes, activation='softmax'))
print(model14.summary())

model14.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist14 = model14.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist14,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3], 2 layers')

hist15 = History()
model15 = Sequential()

model15.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model15.add(BatchNormalization())
model15.add(MaxPooling2D(pool_size=2))
model15.add(Dropout(0.3))

model15.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model15.add(BatchNormalization())
model15.add(MaxPooling2D(pool_size=2))
model15.add(Dropout(0.4))

# model14.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
# model14.add(BatchNormalization())
# model14.add(MaxPooling2D(pool_size=2))
# model14.add(Dropout(0.5))

model15.add(Flatten())
model15.add(Dense(classes, activation='softmax'))
print(model15.summary())

model15.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist15 = model15.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist15,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-4], 2 layers')

hist16 = History()
model16 = Sequential()

model16.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.3))

model16.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.4))

model16.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.5))

model16.add(Conv2D(512, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.5))

model16.add(Flatten())
model16.add(Dense(classes, activation='softmax'))
print(model16.summary())

model16.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist16 = model16.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist16,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3], 4 layers')

hist17 = History()
model17 = Sequential()

model17.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.3))

model17.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.4))

model17.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.5))

model17.add(Conv2D(512, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.5))

model17.add(Flatten())
model17.add(Dense(classes, activation='softmax'))
print(model17.summary())

model17.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist17 = model17.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist17,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3], 4 layers')

hist18 = History()
model18 = Sequential()

model18.add(Conv2D(16, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model18.add(BatchNormalization())
model18.add(MaxPooling2D(pool_size=2))
model18.add(Dropout(0.3))

model18.add(Conv2D(32, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model18.add(BatchNormalization())
model18.add(MaxPooling2D(pool_size=2))
model18.add(Dropout(0.4))

model18.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model18.add(BatchNormalization())
model18.add(MaxPooling2D(pool_size=2))
model18.add(Dropout(0.5))

model18.add(Flatten())
model18.add(Dense(classes, activation='softmax'))
print(model18.summary())

model18.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist18 = model18.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist18,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')



hist19 = History()
model19 = Sequential()

model19.add(Conv2D(2, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model19.add(BatchNormalization())
model19.add(MaxPooling2D(pool_size=2))
model19.add(Dropout(0.3))

model19.add(Conv2D(4, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model19.add(BatchNormalization())
model19.add(MaxPooling2D(pool_size=2))
model19.add(Dropout(0.4))

model19.add(Conv2D(8, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model19.add(BatchNormalization())
model19.add(MaxPooling2D(pool_size=2))
model19.add(Dropout(0.5))

model19.add(Flatten())
model19.add(Dense(classes, activation='softmax'))
print(model19.summary())

model19.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist19 = model19.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist19,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')

# Add some gaussian blur to images
def augment_data(img,k=11,k2=11,sigma=0.5):
    import cv2
    smooth_img = cv2.GaussianBlur(img, (k,k2), sigma)
    return smooth_img

# convolve augmentation
# calculates the horizontal gradient of images using some simple finite kernel [-1, 0, 1]
from scipy.signal import convolve2d
def augment_data2(img):
  # first add the gaussian blur to smooth the images
  img2 = augment_data(img)
  # then convolve
  horz_grad = convolve2d(img2, [[-1, 0, 1]], mode='same')
  return horz_grad

datagen = ImageDataGenerator(
    rotation_range=5,
    zoom_range=2,
    horizontal_flip=True,
    zca_whitening=True)

aug_imgs = []
# Iterate through the images
for i in range(0,xtrain_.shape[0]):
  # Call the function
  aug_img = augment_data2(xtrain_[i])
  
  # Append into the lists
  aug_imgs.append(aug_img)

# Turn into np arrays
aug_imgs = np.array(aug_imgs)

visualise_data(aug_imgs,y_train)

aug_imgs = aug_imgs.reshape(50000,32,32,1)

datagen.fit(aug_imgs)



model20 = Sequential()

model20.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(1e-4)))
model20.add(BatchNormalization())
model20.add(MaxPooling2D(pool_size=2))
model20.add(Dropout(0.5))

model20.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(1e-4)))
model20.add(BatchNormalization())
model20.add(MaxPooling2D(pool_size=2))
model20.add(Dropout(0.5))

model20.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(1e-4)))
model20.add(BatchNormalization())
model20.add(MaxPooling2D(pool_size=2))
model20.add(Dropout(0.5))

model20.add(Flatten())
model20.add(Dense(classes, activation='softmax'))
print(model20.summary())



model20.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist20 = model20.fit(aug_imgs, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist20,'Aug CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')

model21 = Sequential()

model21.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model21.add(BatchNormalization())
model21.add(MaxPooling2D(pool_size=2))
model21.add(Dropout(0.3))

model21.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model21.add(BatchNormalization())
model21.add(MaxPooling2D(pool_size=2))
model21.add(Dropout(0.4))

model21.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model21.add(BatchNormalization())
model21.add(MaxPooling2D(pool_size=2))
model21.add(Dropout(0.5))

model21.add(Flatten())
model21.add(Dense(classes, activation='softmax'))
print(model21.summary())

model21.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist21 = model21.fit(aug_imgs, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist21,'Aug CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')

model22 = Sequential()

model22.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model22.add(BatchNormalization())
model22.add(MaxPooling2D(pool_size=2))
model22.add(Dropout(0.3))

model22.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model22.add(BatchNormalization())
model22.add(MaxPooling2D(pool_size=2))
model22.add(Dropout(0.4))

model22.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model22.add(BatchNormalization())
model22.add(MaxPooling2D(pool_size=2))
model22.add(Dropout(0.5))

model22.add(Flatten())
model22.add(Dense(classes, activation='softmax'))
print(model22.summary())

model22.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist22 = model22.fit(aug_imgs, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)


# -*- coding: utf-8 -*-
"""Neural Networks Final.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/10Lsy5WmktihXzjDCz2eNCoXLHk4o0X1v

# Neural Networks Coursework Assignment
## Multi-class Classification: CIFAR-10 Dataset
### Candidate No. 215921 
### Dr. <NAME>

# Data Loading and Visualisation

1. Import all necessary libraries, including the dataset
"""

from matplotlib import pyplot as plt
import numpy as np
import tensorflow as tf
import cv2
import keras
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping
from tensorflow.keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout,Activation,BatchNormalization
from keras.callbacks import ModelCheckpoint, History
from keras.optimizers import Adam, Adadelta, SGD, RMSprop
from keras.models import load_model
from keras.utils.vis_utils import plot_model
from keras import regularizers
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from keras.regularizers import l2,l1
# Dataset library
from keras.datasets import cifar10

"""2. Load the dataset as a tuple of numpy arrays"""

(x_train,y_train),(x_test,y_test) = cifar10.load_data()

# 50,000 training and 10,000 test, (32x32x3 for RGB images, 1 for labels being an integer 0-9)
# each label denotes one of 10 classes respectively (0-9) => airplane, automobile, bird, cat, deer, dog, frog,horse, ship, truck
print('Training images shape:',x_train.shape)
print('Training labels shape:',y_train.shape)
print('Test images shape:',x_test.shape)
print('Test labels shape:',y_test.shape)

# Random data visualisation function that prints the labels as well.
def visualise_data(data,y_data):
    f, axarr = plt.subplots(2,2)

    img1 = np.random.randint(0, data.shape[0])
    axarr[0,0].imshow(data[img1])

    img2 = np.random.randint(0, data.shape[0])
    axarr[0,1].imshow(data[img2])

    img3 = np.random.randint(0, data.shape[0])
    axarr[1,0].imshow(data[img3])

    img4 = np.random.randint(0, data.shape[0])
    axarr[1,1].imshow(data[img4])

    # show the label of each image above it
    label1 = y_data[img1][0]
    axarr[0,0].title.set_text(label1)
    label2 = y_data[img2][0]
    axarr[0,1].title.set_text(label2)
    label3 = y_data[img3][0]
    axarr[1,0].title.set_text(label3)
    label4 = y_data[img4][0]
    axarr[1,1].title.set_text(label4)

    plt.tight_layout()

# Visualise the training data
visualise_data(x_train,y_train)

"""# Data Preprocessing

3. Convert the image data to 32-bit precision and normalise it between 0-1
"""

x_train_norm = x_train.astype('float32')/255.0
x_test_norm = x_test.astype('float32')/255.0

"""4. Convert the image data to grayscale"""

xtrain_gray = []
for i in range(0,x_train_norm.shape[0]):
  xtrain_gray.append(cv2.cvtColor(x_train_norm[i],cv2.COLOR_BGR2GRAY))
xtrain_gray = np.array(xtrain_gray)
xtrain_gray.shape

xtest_gray = []
for i in range(0,x_test_norm.shape[0]):
  xtest_gray.append(cv2.cvtColor(x_test_norm[i],cv2.COLOR_BGR2GRAY))
xtest_gray = np.array(xtest_gray)
xtest_gray.shape

plt.imshow(xtrain_gray[0],cmap='gray')

"""5. Apply one-hot encoding for the label data """

enc_ytrain = to_categorical(y_train)
enc_ytest = to_categorical(y_test)

y_train[0]

"""It can be seen that the first element of ytrain has a label of 6. Thus, it is categorised at the 6th index as seen below."""

enc_ytrain[0]

"""# Building the model"""

def plot_acc_loss(hist,data_title=''):
    # loss plot 
    plt.figure(figsize=(16,5))
    plt.subplot(1, 2, 1)
    plt.suptitle(data_title, fontsize=10)
    plt.ylabel('Loss', fontsize=16)
    plt.xlabel('Epochs', fontsize=16)
    plt.plot(hist.history['loss'], color='g', label='Training Loss')
    plt.plot(hist.history['val_loss'], color='r', label='Validation Loss')
    plt.legend(loc='upper right')

    # accuracy plot
    plt.subplot(1, 2, 2)
    plt.ylabel('Accuracy', fontsize=16)
    plt.xlabel('Epochs', fontsize=16)
    plt.plot(hist.history['accuracy'], color='g', label='Training Accuracy')
    plt.plot(hist.history['val_accuracy'], color='r', label='Validation Accuracy')
    plt.legend(loc='lower right')
    plt.show()

"""#### 1. Model without Batch Normalisation"""

hist = History()

input_shape = (32,32,1)
epochs = 200
batch_size = 128
classes = 10
xtrain_ = xtrain_gray.reshape(50000,32,32,1)

model1 = Sequential()

model1.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
#model1.add(BatchNormalization())
model1.add(MaxPooling2D(pool_size=2))
model1.add(Dropout(0.3))

model1.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
#model1.add(BatchNormalization())
model1.add(MaxPooling2D(pool_size=2))
model1.add(Dropout(0.4))

model1.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
#model1.add(BatchNormalization())
model1.add(MaxPooling2D(pool_size=2))
model1.add(Dropout(0.5))

model1.add(Flatten())
model1.add(Dense(classes, activation='softmax'))
print(model1.summary())

model1.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist = model1.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist,'Original CIFAR-10 Data (Grayscale, Normalised) [Without Batch Normalisation]')

hist2 = History()

model2 = Sequential()

model2.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=2))
model2.add(Dropout(0.3))

model2.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=2))
model2.add(Dropout(0.4))

model2.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=2))
model2.add(Dropout(0.5))

model2.add(Flatten())
model2.add(Dense(classes, activation='softmax'))
print(model2.summary())

model2.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist2 = model2.fit(xtrain_, enc_ytrain,batch_size=batch_size,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist2,'Original CIFAR-10 Data (Grayscale, Normalised) [With Batch Normalisation]')

X_train, X_val, y_train, y_val = train_test_split(xtrain_, enc_ytrain,shuffle=True)

hist3 = History()

model3 = Sequential()

model3.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model3.add(BatchNormalization())
model3.add(MaxPooling2D(pool_size=2))
model3.add(Dropout(0.3))

model3.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model3.add(BatchNormalization())
model3.add(MaxPooling2D(pool_size=2))
model3.add(Dropout(0.4))

model3.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model3.add(BatchNormalization())
model3.add(MaxPooling2D(pool_size=2))
model3.add(Dropout(0.5))

model3.add(Flatten())
model3.add(Dense(classes, activation='softmax'))
print(model3.summary())

model3.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist3 = model3.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist3,'Original CIFAR-10 Data (Grayscale, Normalised) [Learning Rate = 1e-3]')

hist4 = History()

model4 = Sequential()

model4.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model4.add(BatchNormalization())
model4.add(MaxPooling2D(pool_size=2))
model4.add(Dropout(0.3))

model4.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model4.add(BatchNormalization())
model4.add(MaxPooling2D(pool_size=2))
model4.add(Dropout(0.4))

model4.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model4.add(BatchNormalization())
model4.add(MaxPooling2D(pool_size=2))
model4.add(Dropout(0.5))

model4.add(Flatten())
model4.add(Dense(classes, activation='softmax'))
print(model4.summary())

model4.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-5), metrics=['accuracy','CategoricalAccuracy'])

hist4 = model4.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist4,'Original CIFAR-10 Data (Grayscale, Normalised) [Learning Rate = 1e-5]')

hist5 = History()
model5 = Sequential()

model5.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model5.add(BatchNormalization())
model5.add(MaxPooling2D(pool_size=2))
model5.add(Dropout(0.3))

model5.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model5.add(BatchNormalization())
model5.add(MaxPooling2D(pool_size=2))
model5.add(Dropout(0.4))

model5.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model5.add(BatchNormalization())
model5.add(MaxPooling2D(pool_size=2))
model5.add(Dropout(0.5))

model5.add(Flatten())
model5.add(Dense(classes, activation='softmax'))
print(model5.summary())

model5.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-2), metrics=['accuracy','CategoricalAccuracy'])

hist5 = model5.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist5,'Original CIFAR-10 Data (Grayscale, Normalised) [Learning Rate = 1e-2]')

hist6 = History()
model6 = Sequential()

model6.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model6.add(BatchNormalization())
model6.add(MaxPooling2D(pool_size=2))
model6.add(Dropout(0.3))

model6.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model6.add(BatchNormalization())
model6.add(MaxPooling2D(pool_size=2))
model6.add(Dropout(0.4))

model6.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model6.add(BatchNormalization())
model6.add(MaxPooling2D(pool_size=2))
model6.add(Dropout(0.5))

model6.add(Flatten())
model6.add(Dense(classes, activation='softmax'))
print(model6.summary())

model6.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist6 = model6.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist6,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-4]')

hist7 = History()
model7 = Sequential()

model7.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model7.add(BatchNormalization())
model7.add(MaxPooling2D(pool_size=2))
model7.add(Dropout(0.3))

model7.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model7.add(BatchNormalization())
model7.add(MaxPooling2D(pool_size=2))
model7.add(Dropout(0.4))

model7.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model7.add(BatchNormalization())
model7.add(MaxPooling2D(pool_size=2))
model7.add(Dropout(0.5))

model7.add(Flatten())
model7.add(Dense(classes, activation='softmax'))
print(model7.summary())

model7.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-5), metrics=['accuracy','CategoricalAccuracy'])

hist7 = model7.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist7,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-5]')

hist8 = History()
model8 = Sequential()

model8.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model8.add(BatchNormalization())
model8.add(MaxPooling2D(pool_size=2))
model8.add(Dropout(0.3))

model8.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model8.add(BatchNormalization())
model8.add(MaxPooling2D(pool_size=2))
model8.add(Dropout(0.4))

model8.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model8.add(BatchNormalization())
model8.add(MaxPooling2D(pool_size=2))
model8.add(Dropout(0.5))

model8.add(Flatten())
model8.add(Dense(classes, activation='softmax'))
print(model8.summary())

model8.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist8 = model8.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist8,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-3]')

hist9 = History()
model9 = Sequential()

model9.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model9.add(BatchNormalization())
model9.add(MaxPooling2D(pool_size=2))
model9.add(Dropout(0.3))

model9.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model9.add(BatchNormalization())
model9.add(MaxPooling2D(pool_size=2))
model9.add(Dropout(0.4))

model9.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model9.add(BatchNormalization())
model9.add(MaxPooling2D(pool_size=2))
model9.add(Dropout(0.5))

model9.add(Flatten())
model9.add(Dense(classes, activation='softmax'))
print(model9.summary())

model9.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1.0e-2), metrics=['accuracy','CategoricalAccuracy'])

hist9 = model9.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist9,'Original CIFAR-10 Data (Grayscale, Normalised) [SGD, Learning Rate = 1e-2]')

hist10 = History()
model10 = Sequential()

model10.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model10.add(BatchNormalization())
model10.add(MaxPooling2D(pool_size=2))
model10.add(Dropout(0.3))

model10.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model10.add(BatchNormalization())
model10.add(MaxPooling2D(pool_size=2))
model10.add(Dropout(0.4))

model10.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model10.add(BatchNormalization())
model10.add(MaxPooling2D(pool_size=2))
model10.add(Dropout(0.5))

model10.add(Flatten())
model10.add(Dense(classes, activation='softmax'))
print(model10.summary())

model10.save('model.h5')

model10.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-2), metrics=['accuracy','CategoricalAccuracy'])

hist10 = model10.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist10,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-2]')

hist11 = History()
model11 = Sequential()

model11.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model11.add(BatchNormalization())
model11.add(MaxPooling2D(pool_size=2))
model11.add(Dropout(0.3))

model11.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model11.add(BatchNormalization())
model11.add(MaxPooling2D(pool_size=2))
model11.add(Dropout(0.4))

model11.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model11.add(BatchNormalization())
model11.add(MaxPooling2D(pool_size=2))
model11.add(Dropout(0.5))

model11.add(Flatten())
model11.add(Dense(classes, activation='softmax'))
print(model11.summary())

model11.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist11 = model11.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist11,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-3]')



hist12 = History()
model12 = Sequential()

model12.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model12.add(BatchNormalization())
model12.add(MaxPooling2D(pool_size=2))
model12.add(Dropout(0.3))

model12.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model12.add(BatchNormalization())
model12.add(MaxPooling2D(pool_size=2))
model12.add(Dropout(0.4))

model12.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model12.add(BatchNormalization())
model12.add(MaxPooling2D(pool_size=2))
model12.add(Dropout(0.5))

model12.add(Flatten())
model12.add(Dense(classes, activation='softmax'))
print(model12.summary())

model12.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist12 = model12.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist12,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-4]')





hist13 = History()
model13 = Sequential()

model13.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model13.add(BatchNormalization())
model13.add(MaxPooling2D(pool_size=2))
model13.add(Dropout(0.3))

model13.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model13.add(BatchNormalization())
model13.add(MaxPooling2D(pool_size=2))
model13.add(Dropout(0.4))

model13.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model13.add(BatchNormalization())
model13.add(MaxPooling2D(pool_size=2))
model13.add(Dropout(0.5))

model13.add(Flatten())
model13.add(Dense(classes, activation='softmax'))
print(model13.summary())

model13.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=1.0e-5), metrics=['accuracy','CategoricalAccuracy'])

hist13 = model13.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist13,'Original CIFAR-10 Data (Grayscale, Normalised) [RMSprop, Learning Rate = 1e-5]')



hist14 = History()
model14 = Sequential()

model14.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model14.add(BatchNormalization())
model14.add(MaxPooling2D(pool_size=2))
model14.add(Dropout(0.3))

model14.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model14.add(BatchNormalization())
model14.add(MaxPooling2D(pool_size=2))
model14.add(Dropout(0.4))

# model14.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
# model14.add(BatchNormalization())
# model14.add(MaxPooling2D(pool_size=2))
# model14.add(Dropout(0.5))

model14.add(Flatten())
model14.add(Dense(classes, activation='softmax'))
print(model14.summary())

model14.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist14 = model14.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist14,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3], 2 layers')

hist15 = History()
model15 = Sequential()

model15.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model15.add(BatchNormalization())
model15.add(MaxPooling2D(pool_size=2))
model15.add(Dropout(0.3))

model15.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model15.add(BatchNormalization())
model15.add(MaxPooling2D(pool_size=2))
model15.add(Dropout(0.4))

# model14.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
# model14.add(BatchNormalization())
# model14.add(MaxPooling2D(pool_size=2))
# model14.add(Dropout(0.5))

model15.add(Flatten())
model15.add(Dense(classes, activation='softmax'))
print(model15.summary())

model15.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist15 = model15.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist15,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-4], 2 layers')

hist16 = History()
model16 = Sequential()

model16.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.3))

model16.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.4))

model16.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.5))

model16.add(Conv2D(512, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model16.add(BatchNormalization())
model16.add(MaxPooling2D(pool_size=2))
model16.add(Dropout(0.5))

model16.add(Flatten())
model16.add(Dense(classes, activation='softmax'))
print(model16.summary())

model16.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist16 = model16.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist16,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3], 4 layers')

hist17 = History()
model17 = Sequential()

model17.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.3))

model17.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.4))

model17.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.5))

model17.add(Conv2D(512, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model17.add(BatchNormalization())
model17.add(MaxPooling2D(pool_size=2))
model17.add(Dropout(0.5))

model17.add(Flatten())
model17.add(Dense(classes, activation='softmax'))
print(model17.summary())

model17.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist17 = model17.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist17,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3], 4 layers')

hist18 = History()
model18 = Sequential()

model18.add(Conv2D(16, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model18.add(BatchNormalization())
model18.add(MaxPooling2D(pool_size=2))
model18.add(Dropout(0.3))

model18.add(Conv2D(32, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model18.add(BatchNormalization())
model18.add(MaxPooling2D(pool_size=2))
model18.add(Dropout(0.4))

model18.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model18.add(BatchNormalization())
model18.add(MaxPooling2D(pool_size=2))
model18.add(Dropout(0.5))

model18.add(Flatten())
model18.add(Dense(classes, activation='softmax'))
print(model18.summary())

model18.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist18 = model18.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist18,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')



hist19 = History()
model19 = Sequential()

model19.add(Conv2D(2, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model19.add(BatchNormalization())
model19.add(MaxPooling2D(pool_size=2))
model19.add(Dropout(0.3))

model19.add(Conv2D(4, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model19.add(BatchNormalization())
model19.add(MaxPooling2D(pool_size=2))
model19.add(Dropout(0.4))

model19.add(Conv2D(8, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model19.add(BatchNormalization())
model19.add(MaxPooling2D(pool_size=2))
model19.add(Dropout(0.5))

model19.add(Flatten())
model19.add(Dense(classes, activation='softmax'))
print(model19.summary())

model19.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist19 = model19.fit(xtrain_, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist19,'Original CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')

# Add some gaussian blur to images
def augment_data(img,k=11,k2=11,sigma=0.5):
    import cv2
    smooth_img = cv2.GaussianBlur(img, (k,k2), sigma)
    return smooth_img

# convolve augmentation
# calculates the horizontal gradient of images using some simple finite kernel [-1, 0, 1]
from scipy.signal import convolve2d
def augment_data2(img):
  # first add the gaussian blur to smooth the images
  img2 = augment_data(img)
  # then convolve
  horz_grad = convolve2d(img2, [[-1, 0, 1]], mode='same')
  return horz_grad

datagen = ImageDataGenerator(
    rotation_range=5,
    zoom_range=2,
    horizontal_flip=True,
    zca_whitening=True)

aug_imgs = []
# Iterate through the images
for i in range(0,xtrain_.shape[0]):
  # Call the function
  aug_img = augment_data2(xtrain_[i])
  
  # Append into the lists
  aug_imgs.append(aug_img)

# Turn into np arrays
aug_imgs = np.array(aug_imgs)

visualise_data(aug_imgs,y_train)

aug_imgs = aug_imgs.reshape(50000,32,32,1)

datagen.fit(aug_imgs)



model20 = Sequential()

model20.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(1e-4)))
model20.add(BatchNormalization())
model20.add(MaxPooling2D(pool_size=2))
model20.add(Dropout(0.5))

model20.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(1e-4)))
model20.add(BatchNormalization())
model20.add(MaxPooling2D(pool_size=2))
model20.add(Dropout(0.5))

model20.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(1e-4)))
model20.add(BatchNormalization())
model20.add(MaxPooling2D(pool_size=2))
model20.add(Dropout(0.5))

model20.add(Flatten())
model20.add(Dense(classes, activation='softmax'))
print(model20.summary())



model20.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist20 = model20.fit(aug_imgs, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist20,'Aug CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')

model21 = Sequential()

model21.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model21.add(BatchNormalization())
model21.add(MaxPooling2D(pool_size=2))
model21.add(Dropout(0.3))

model21.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model21.add(BatchNormalization())
model21.add(MaxPooling2D(pool_size=2))
model21.add(Dropout(0.4))

model21.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same'))
model21.add(BatchNormalization())
model21.add(MaxPooling2D(pool_size=2))
model21.add(Dropout(0.5))

model21.add(Flatten())
model21.add(Dense(classes, activation='softmax'))
print(model21.summary())

model21.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-4), metrics=['accuracy','CategoricalAccuracy'])

hist21 = model21.fit(aug_imgs, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)

plot_acc_loss(hist21,'Aug CIFAR-10 Data (Grayscale, Normalised) [Adam, Learning Rate = 1e-3]')

model22 = Sequential()

model22.add(Conv2D(64, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model22.add(BatchNormalization())
model22.add(MaxPooling2D(pool_size=2))
model22.add(Dropout(0.3))

model22.add(Conv2D(128, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model22.add(BatchNormalization())
model22.add(MaxPooling2D(pool_size=2))
model22.add(Dropout(0.4))

model22.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same',kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
model22.add(BatchNormalization())
model22.add(MaxPooling2D(pool_size=2))
model22.add(Dropout(0.5))

model22.add(Flatten())
model22.add(Dense(classes, activation='softmax'))
print(model22.summary())

model22.compile(loss='categorical_crossentropy', optimizer=Adam(lr=1.0e-3), metrics=['accuracy','CategoricalAccuracy'])

hist22 = model22.fit(aug_imgs, enc_ytrain,batch_size=128,epochs=100,validation_split=0.3,shuffle=True)
# -*- coding: utf-8 -*- Neural Networks Final.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/10Lsy5WmktihXzjDCz2eNCoXLHk4o0X1v  # Neural Networks Coursework Assignment ## Multi-class Classification: CIFAR-10 Dataset ### Candidate No. 215921  ### Dr. <NAME>  # Data Loading and Visualisation  1. Import all necessary libraries, including the dataset # Dataset library 2. Load the dataset as a tuple of numpy arrays # 50,000 training and 10,000 test, (32x32x3 for RGB images, 1 for labels being an integer 0-9) # each label denotes one of 10 classes respectively (0-9) => airplane, automobile, bird, cat, deer, dog, frog,horse, ship, truck # Random data visualisation function that prints the labels as well. # show the label of each image above it # Visualise the training data # Data Preprocessing  3. Convert the image data to 32-bit precision and normalise it between 0-1 4. Convert the image data to grayscale 5. Apply one-hot encoding for the label data It can be seen that the first element of ytrain has a label of 6. Thus, it is categorised at the 6th index as seen below. # Building the model # loss plot # accuracy plot #### 1. Model without Batch Normalisation #model1.add(BatchNormalization()) #model1.add(BatchNormalization()) #model1.add(BatchNormalization()) # model14.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same')) # model14.add(BatchNormalization()) # model14.add(MaxPooling2D(pool_size=2)) # model14.add(Dropout(0.5)) # model14.add(Conv2D(264, kernel_size=3, input_shape=input_shape, activation ="relu",padding='same')) # model14.add(BatchNormalization()) # model14.add(MaxPooling2D(pool_size=2)) # model14.add(Dropout(0.5)) # Add some gaussian blur to images # convolve augmentation # calculates the horizontal gradient of images using some simple finite kernel [-1, 0, 1] # first add the gaussian blur to smooth the images # then convolve # Iterate through the images # Call the function # Append into the lists # Turn into np arrays