<reponame>Takahiro-Funahashi/o-reilly-deep-learning-clone
# coding: utf-8
import sys
import numpy as np
import matplotlib.pyplot as plt

sys.path.append('./common/')
sys.path.append('./dataset/')

if __name__ == '__main__':
    from mnist import load_mnist
    from multi_layer_net import MultiLayerNet
    from util import shuffle_dataset
    from trainer import Trainer

    (x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)

    # 高速化のため訓練データの削減
    x_train = x_train[:500]
    t_train = t_train[:500]

    # 検証データの分離
    validation_rate = 0.20
    validation_num = int(x_train.shape[0] * validation_rate)
    x_train, t_train = shuffle_dataset(x_train, t_train)
    x_val = x_train[:validation_num]
    t_val = t_train[:validation_num]
    x_train = x_train[validation_num:]
    t_train = t_train[validation_num:]

    def __train(lr, weight_decay, epocs=50):
        network = MultiLayerNet(
            input_size=784, hidden_size_list=[100, 100, 100, 100, 100, 100],
            output_size=10, weight_decay_lambda=weight_decay)
        trainer = Trainer(
            network, x_train, t_train, x_val, t_val,
            epochs=epocs, mini_batch_size=100,
            optimizer='sgd', optimizer_param={'lr': lr}, verbose=False)
        trainer.train()

        return trainer.test_acc_list, trainer.train_acc_list

    # ハイパーパラメータのランダム探索======================================
    optimization_trial = 100
    results_val = {}
    results_train = {}
    for _ in range(optimization_trial):
        # 探索したハイパーパラメータの範囲を指定===============
        weight_decay = 10 ** np.random.uniform(-8, -4)
        lr = 10 ** np.random.uniform(-6, -2)
        # ================================================

        val_acc_list, train_acc_list = __train(lr, weight_decay)
        print("val acc:" + str(val_acc_list[-1]) + " | lr:" +
              str(lr) + ", weight decay:" + str(weight_decay))
        key = "lr:" + str(lr) + ", weight decay:" + str(weight_decay)
        results_val[key] = val_acc_list
        results_train[key] = train_acc_list

    # グラフの描画========================================================
    print("=========== Hyper-Parameter Optimization Result ===========")
    graph_draw_num = 20
    col_num = 5
    row_num = int(np.ceil(graph_draw_num / col_num))
    i = 0

    for key, val_acc_list in sorted(
            results_val.items(), key=lambda x: x[1][-1], reverse=True):
        print("Best-" + str(i+1) + "(val acc:" +
              str(val_acc_list[-1]) + ") | " + key)

        plt.subplot(row_num, col_num, i+1)
        plt.title("Best-" + str(i+1))
        plt.ylim(0.0, 1.0)
        if i % 5:
            plt.yticks([])
        plt.xticks([])
        x = np.arange(len(val_acc_list))
        plt.plot(x, val_acc_list)
        plt.plot(x, results_train[key], "--")
        i += 1

        if i >= graph_draw_num:
            break

    plt.show()

# coding: utf-8
import sys
import numpy as np
import matplotlib.pyplot as plt

sys.path.append('./common/')
sys.path.append('./dataset/')

if __name__ == '__main__':
    from mnist import load_mnist
    from multi_layer_net import MultiLayerNet
    from util import shuffle_dataset
    from trainer import Trainer

    (x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)

    # 高速化のため訓練データの削減
    x_train = x_train[:500]
    t_train = t_train[:500]

    # 検証データの分離
    validation_rate = 0.20
    validation_num = int(x_train.shape[0] * validation_rate)
    x_train, t_train = shuffle_dataset(x_train, t_train)
    x_val = x_train[:validation_num]
    t_val = t_train[:validation_num]
    x_train = x_train[validation_num:]
    t_train = t_train[validation_num:]

    def __train(lr, weight_decay, epocs=50):
        network = MultiLayerNet(
            input_size=784, hidden_size_list=[100, 100, 100, 100, 100, 100],
            output_size=10, weight_decay_lambda=weight_decay)
        trainer = Trainer(
            network, x_train, t_train, x_val, t_val,
            epochs=epocs, mini_batch_size=100,
            optimizer='sgd', optimizer_param={'lr': lr}, verbose=False)
        trainer.train()

        return trainer.test_acc_list, trainer.train_acc_list

    # ハイパーパラメータのランダム探索======================================
    optimization_trial = 100
    results_val = {}
    results_train = {}
    for _ in range(optimization_trial):
        # 探索したハイパーパラメータの範囲を指定===============
        weight_decay = 10 ** np.random.uniform(-8, -4)
        lr = 10 ** np.random.uniform(-6, -2)
        # ================================================

        val_acc_list, train_acc_list = __train(lr, weight_decay)
        print("val acc:" + str(val_acc_list[-1]) + " | lr:" +
              str(lr) + ", weight decay:" + str(weight_decay))
        key = "lr:" + str(lr) + ", weight decay:" + str(weight_decay)
        results_val[key] = val_acc_list
        results_train[key] = train_acc_list

    # グラフの描画========================================================
    print("=========== Hyper-Parameter Optimization Result ===========")
    graph_draw_num = 20
    col_num = 5
    row_num = int(np.ceil(graph_draw_num / col_num))
    i = 0

    for key, val_acc_list in sorted(
            results_val.items(), key=lambda x: x[1][-1], reverse=True):
        print("Best-" + str(i+1) + "(val acc:" +
              str(val_acc_list[-1]) + ") | " + key)

        plt.subplot(row_num, col_num, i+1)
        plt.title("Best-" + str(i+1))
        plt.ylim(0.0, 1.0)
        if i % 5:
            plt.yticks([])
        plt.xticks([])
        x = np.arange(len(val_acc_list))
        plt.plot(x, val_acc_list)
        plt.plot(x, results_train[key], "--")
        i += 1

        if i >= graph_draw_num:
            break

    plt.show()
# coding: utf-8 # 高速化のため訓練データの削減 # 検証データの分離 # ハイパーパラメータのランダム探索====================================== # 探索したハイパーパラメータの範囲を指定=============== # ================================================ # グラフの描画========================================================