def load_mnist_data(nb_training, nb_test, preprocess=False, flatten=True, one_hot=True):
    # 自动检查数据，如果数据文件不存在则会先自动下载
    download_mnist_data()
    training_x = mnist_decoder.load_train_images(num_data=nb_training)
    training_y = mnist_decoder.load_train_labels(num_data=nb_training)
    test_x = mnist_decoder.load_test_images(num_data=nb_test)
    test_y = mnist_decoder.load_test_labels(num_data=nb_test)

    if preprocess:
        training_x /= 255.
        test_x /= 255.

    if flatten:
        training_x = training_x.reshape(training_x.shape[0], 784)
        test_x = test_x.reshape(test_x.shape[0], 784)
    else:
        training_x = training_x.reshape((training_x.shape[0], 1, training_x.shape[1], training_x.shape[2]))
        test_x = test_x.reshape((test_x.shape[0], 1, test_x.shape[1], test_x.shape[2]))

    if one_hot:
        training_y = np_utils.to_one_hot(training_y, 10)
        test_y = np_utils.to_one_hot(test_y, 10)

    return (training_x, training_y), (test_x, test_y) class Dense(Layer):
    """
    全连接层。
    """

    layer_type = 'Dense'

    def __init__(self, output_dim, input_dim=None, init='uniform', **kwargs):
        super(Dense, self).__init__()
        # Layer.__init__(self)
        # 构造方法传入参数
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.init = initializations.get(init)

        # 输入输出数据及其形状
        self.input = None
        self.output = None
        self.d_input = None
        self.d_output = None
        self.input_shape = [None, input_dim]
        self.output_shape = [None, output_dim]

        # 中间计算结果
        self.mid = {}

        # 模型参数
        self.w = None
        self.b = None

        # 模型基本成员
        self.trainable = True
        self.trainable_params = []
        self.grads = []

        self.previous_layer = None
        self.latter_layer = None

        self.mode = "experiment"
        self.mid = {}
        self.config = {}

    def build(self):
        # 如果是第一层
        if not self.previous_layer:
            # 如果没有给定输入数据的维数，则报错
            if not self.input_dim:
                raise Exception('BuildError : 全连接层为第一层'
                                '时必须在构造方法中提供input_dim'
                                '参数!')
        # 如果不是第一层
        else:
            self.input_dim = self.previous_layer.output_shape[-1]

        # 设置层的输入输出形状
        self.input_shape[1] = self.input_dim

        # 初始化参数
        self.w = self.init((self.input_dim, self.output_dim))
        self.b = initializations.zeros((1, self.output_dim))
        self.trainable_params = [self.w, self.b]

    def forward_train(self, _input):
        self.input = _input
        # 前向计算
        mul = MulGate.forward(self.input, self.w)
        add = AddGate.forward(mul, self.b)
        self.output = add
        # 保留中间计算结果
        self.mid['mul'] = mul
        return self.output

    def forward_test(self, _input):
        self.input = _input
        # 前向计算
        mul = MulGate.forward(self.input, self.w)
        add = AddGate.forward(mul, self.b)
        self.output = add
        # 删除中间计算结果
        del mul
        return self.output

    def backward(self, _d_output):
        self.d_output = _d_output
        d_add = self.d_output
        mul = self.mid.get('mul')
        # 反向传播
        d_mul, d_b = AddGate.backward(mul, self.b, d_add)
        self.d_input, d_w = MulGate.backward(self.input, self.w, d_mul)
        # 将需要更新的参数的梯度放在grads列表中
        self.grads = [d_w, d_b]
        return self.d_input class Activation(Layer):
    """
    激活层。
    """

    layer_type = 'Activation'

    def __init__(self, activation, **kwargs):
        super(Activation, self).__init__()

        # 构造方法传入参数
        self.activation = activations.get(activation)

        # 输入输出数据及其形状
        self.input = None
        self.output = None
        self.d_input = None
        self.d_output = None
        self.input_shape = [None, None]
        self.output_shape = [None, None]

        # 模型基本成员
        self.trainable = False
        self.trainable_params = []
        self.grads = []

        self.previous_layer = None
        self.latter_layer = None

        self.mode = "experiment"
        self.mid = {}
        self.config = {}

    def build(self):
        # 如果是第一层
        if not self.previous_layer:
            raise Exception('BuildError: 暂时不支持激活层为第一层!')
        self.input_shape = self.previous_layer.output_shape
        self.output_shape = self.input_shape

    def forward_train(self, _input):
        self.input = _input
        self.output = self.activation.forward(self.input)
        return self.output

    def forward_test(self, _input):
        self.input = _input
        self.output = self.activation.forward(self.input)
        return self.output

    def backward(self, _d_output):
        self.d_output = _d_output
        self.d_input = self.activation.backward(self.input, self.output, self.d_output)
        return self.d_input class Convolution2D(Layer):
    """
    卷积层。
    """

    layer_type = 'Convolution2D'

    def __init__(self, nb_kernel, kernel_height, kernel_width, input_shape=None,
                 init='glorot_normal', activation='linear', stride=1, **kwargs):
        super(Convolution2D, self).__init__()

        # 构造方法传入参数
        self.nb_kernel = nb_kernel
        self.kernel_height = kernel_height
        self.kernel_width = kernel_width
        self.CHW_shape = input_shape
        self.init = initializations.get(init)
        self.activation = activations.get(activation)
        self.stride = stride

        # 输入输出数据及其形状
        self.input = None
        self.output = None
        self.d_input = None
        self.d_output = None
        self.input_shape = [None, None, None, None]
        self.output_shape = [None, None, None, None]

        self.w_shape = [None, None, None, None]

        # 模型参数
        self.w = None
        self.b = None

        # 模型基本成员
        self.trainable = True
        self.trainable_params = []
        self.grads = []

        self.previous_layer = None
        self.latter_layer = None

        self.mode = "experiment"
        self.mid = {}
        self.config = {}

    def build(self):
        # ==================================
        # 如果是第一层
        if not self.previous_layer:
            # 如果没有指明输入数据形状，则报错
            if not self.CHW_shape:
                raise Exception('BuildError : 卷积层为第一层时必'
                                '须在构造方法中提供input_shape参数!')
            # 如果指明了输入数据形状
            else:
                self.input_shape[1:] = self.CHW_shape
        # 如果不是第一层
        else:
            self.input_shape = self.previous_layer.output_shape
        # ==================================
        # 计算w_shape以及output_shape
        N, C, H, W = self.input_shape
        KN = self.nb_kernel
        KC = self.input_shape[1]
        KH = self.kernel_height
        KW = self.kernel_width
        CH, CW = get_conv_shape(H, W, KH, KW, self.stride)

        self.w_shape = [KN, KC, KH, KW]
        self.output_shape = [N, KN, CH, CW]

        # 初始化参数
        self.w = self.init(shape=self.w_shape)
        self.b = initializations.zeros(shape=(1, KN, 1, 1))
        self.trainable_params = [self.w, self.b]

    def forward_train(self, _input):
        self.input = _input
        self.output_shape[0] = self.input_shape[0] = self.input.shape[0]

        # 计算形状
        N, C, H, W = self.input_shape
        KN, KC, KH, KW = self.w_shape
        CH, CW = self.output_shape[2:]

        # 准备工作
        x = self.input
        rowing_w = self.w.reshape(KN, KC * KH * KW)

        # 前向计算
        columnize_x = im2col_NCHW(x, KH, KW, self.stride)
        rowing_mul = MulGate.forward(rowing_w, columnize_x)
        # rowing_w:     (KN, C*KH*KW)
        # columnize_x:  (C*KH*KW, N*CH*CW)
        # rowing_mul:   (KN, N*CH*CW)
        # mul:          (N, KN, CH, CW)
        mul = rowing_mul.reshape(KN, N, CH, CW).swapaxes(0, 1)
        add = AddGate.forward(mul, self.b)
        act = self.activation.forward(add)
        self.output = act

        # 保存中间计算结果
        self.mid['rowing_w'] = rowing_w
        self.mid['columnize_x'] = columnize_x
        self.mid['mul'] = mul
        self.mid['add'] = add

        return self.output

    def forward_test(self, _input):
        self.input = _input

        # 计算形状
        N, C, H, W = self.input_shape
        KN, KC, KH, KW = self.w_shape
        CH, CW = self.output_shape[2:]

        # 准备工作
        x = self.input
        rowing_w = self.w.reshape(KN, KH * KW * C)
        # mul: (N, KN, CH, CW)
        mul = np.empty(self.output_shape, dtype=self.input.dtype)

        # 前向计算
        for n in xrange(N):
            # 将输入变形
            x_n = np.array(x[n]).reshape(1, C, H, W)
            # 计算
            # columnize_x_n: (C*KH*KW, 1*CH*CW)
            columnize_x_n = im2col_NCHW(x_n, KH, KW, self.stride)
            # rowing_mul_n: (KN, 1*CH*CW)
            rowing_mul_n = MulGate.forward(rowing_w, columnize_x_n)
            # 将输出变形
            mul_n = rowing_mul_n.reshape(1, KN, CH, CW)
            # 保存
            mul[n] = mul_n

        add = AddGate.forward(mul, self.b)
        del mul

        act = self.activation.forward(add)
        del add

        self.output = act
        return self.output

    def backward(self, _d_output):
        self.d_output = _d_output

        # 形状变量
        N, C, H, W = self.input_shape
        KN, KC, KH, KW = self.w_shape
        CH, CW = self.output_shape[2:]

        # 准备工作
        d_x = np.empty_like(self.input, dtype=self.input.dtype)
        d_w = np.zeros_like(self.w, dtype=np.double)

        # 提取中间计算变量
        rowing_w = self.mid['rowing_w']
        columnize_x = self.mid['columnize_x']
        mul = self.mid['mul']
        add = self.mid['add']
        act = self.output

        # 反向求导
        d_act = self.d_output
        d_add = self.activation.backward(add, act, d_act)
        d_mul, d_b = AddGate.backward(mul, self.b, d_add)
        d_rowing_mul = d_mul.swapaxes(0, 1).reshape(KN, N * CH * CW)
        d_rowing_w, d_columnize_x = MulGate.backward(rowing_w, columnize_x, d_rowing_mul)
        d_x = col2im_NCHW(d_columnize_x, KH, KW, CH, CW, self.stride)

        self.d_input = d_x
        d_w = d_rowing_w.reshape(KN, KC, KH, KW)

        # 保存梯度
        self.grads = [d_w, d_b]
        return self.d_input class Flatten(Layer):
    """
    平铺层，将4维输入(NCHW)转化为2维输出(NM)。
    """

    layer_type = 'Flatten'

    def __init__(self, input_shape=None, **kwargs):
        super(Flatten, self).__init__()

        self.CHW_shape = input_shape  # (C, H, W)格式

        # 输入输出数据及其形状
        self.input = None
        self.output = None
        self.d_input = None
        self.d_output = None
        self.input_shape = [None, None, None, None]
        self.output_shape = [None, None]

        # 模型基本成员
        self.trainable = False
        self.trainable_params = []
        self.grads = []

        self.previous_layer = None
        self.latter_layer = None

        self.mode = "experiment"
        self.mid = {}
        self.config = {}

    def build(self):
        # 如果是第一层
        if not self.previous_layer:
            if not self.CHW_shape:
                raise Exception('BuildError : Flatten为第一层时必'
                                '须在构造方法中提供input_shape参数!')
            else:
                self.input_shape[1:] = self.CHW_shape
        # 如果不是第一层
        else:
            self.input_shape = self.previous_layer.output_shape
        self.output_shape[-1] = np.product(self.input_shape[1:])

    def forward_train(self, _input):
        self.input = _input
        self.output = self.input.reshape(self.output_shape)
        return self.output

    def forward_test(self, _input):
        self.input = _input
        self.output = self.input.reshape(self.output_shape)
        return self.output

    def backward(self, _d_output):
        self.d_output = _d_output
        self.d_input = self.d_output.reshape(self.input_shape)
        return self.d_input class MeanPooling2D(Layer):
    """
    均值池化层
    """

    layer_type = 'MeanPooling2D'

    def __init__(self, pooling_size, input_shape=None, **kwargs):
        super(MeanPooling2D, self).__init__()

        self.pooling_size = pooling_size
        self.CHW_shape = input_shape  # (C, H, W)格式

        # 输入输出数据及其形状
        self.input = None
        self.output = None
        self.d_input = None
        self.d_output = None
        self.input_shape = [None, None, None, None]
        self.output_shape = [None, None, None, None]

        # 模型基本成员
        self.trainable = False
        self.trainable_params = []
        self.grads = []

        self.previous_layer = None
        self.latter_layer = None

        self.mode = "experiment"
        self.mid = {}
        self.config = {}

    def build(self):
        #  如果是第一层
        if not self.previous_layer:
            if not self.CHW_shape:
                raise Exception('BuildError : MeanPoolnig2D层为第一层'
                                '时必须在构造方法中提供input_shape参数!')
            self.input_shape[1:] = self.CHW_shape
        # 如果具有前一层
        else:
            self.input_shape = self.previous_layer.output_shape

        if self.pooling_size[0] != self.pooling_size[1]:
            raise Exception('BuildError : 目前MeanPooling2D层只支持方形采样！')

        N, C, H, W = self.input_shape
        self.output_shape = [N, C, H/self.pooling_size[0], W/self.pooling_size[1]]

    def forward_train(self, _input):
        self.input = _input

        # 计算形状
        N, C, H, W = self.input_shape
        KN, KC, KH, KW = 1, 1, self.pooling_size[0], self.pooling_size[1]
        CH, CW = self.output_shape[2:]
        stride = KH

        # 准备工作
        rowing_w = np.ones(shape=(KN, KC * KH * KW), dtype=self.input.dtype) / (KH * KW)
        columnize_x = np.empty(shape=(KH * KW, N * C * CH * CW), dtype=self.input.dtype)
        # columnize_x: (KH*KW, N*C*CH*CW)

        # 前向计算
        for n in xrange(N):
            for c in xrange(C):
                # 依次计算input[n][c]的im2col_HW结果，并存放在columnize_x中，每个im2col_HW结果依次挨着横着放
                ocol = (n * C + c) * CH * CW
                columnize_x[0:KH*KW, ocol: ocol+CH*CW] = im2col_HW(self.input[n][c], KH, KW, stride)

        # rowing_output: (KN, N*C*CH*CW)
        # output: (N, C, CH, CW) = (N, C, H/KH, W/KW)
        rowing_output = MulGate.forward(rowing_w, columnize_x)
        self.output = rowing_output.reshape(self.output_shape)

        self.mid['rowing_w'] = rowing_w
        self.mid['columnize_x'] = columnize_x

        return self.output

    def forward_test(self, _input):
        self.input = _input

        N, C, H, W = self.input_shape
        KN, KC, KH, KW = 1, 1, self.pooling_size[0], self.pooling_size[1]
        CH, CW = self.output_shape[2:]
        stride = KH

        rowing_w = np.ones(shape=(KN, KC * KH * KW), dtype=self.input.dtype) / (KH * KW)
        columnize_x = np.empty(shape=(KH * KW, N * C * CH * CW), dtype=self.input.dtype)

        for n in xrange(N):
            for c in xrange(C):
                ocol = (n * C + c) * CH * CW
                columnize_x[0:KH * KW, ocol: ocol + CH * CW] = im2col_HW(self.input[n][c], KH, KW, stride)

        rowing_output = MulGate.forward(rowing_w, columnize_x)
        del columnize_x

        self.output = rowing_output.reshape(self.output_shape)
        return self.output

    def backward(self, _d_output):
        self.d_output = _d_output

        N, C, H, W = self.input_shape
        KN, KC, KH, KW = 1, 1, self.pooling_size[0], self.pooling_size[1]
        CH, CW = self.output_shape[2:]
        stride = KH

        rowing_w = self.mid['rowing_w']
        columnize_x = self.mid['columnize_x']
        d_input = np.empty(shape=self.input_shape, dtype=self.input.dtype)

        d_rowing_output = self.d_output.reshape(1, N * C * CH * CW)
        _, d_columnize_x = MulGate.backward(rowing_w, columnize_x, d_rowing_output)

        for n in xrange(N):
            for c in xrange(C):
                ocol = (n * C + c) * CH * CW
                d_input[n][c] = col2im_HW(d_columnize_x[0:KH*KW, ocol: ocol+CH*CW], KH, KW, CH, CW, stride)

        self.d_input = d_input
        return self.d_input class Sequential(object):
    """一种由网络层线性堆叠而成的模型。

    # Arguments
        layers: 由网络层对象组成的列表。

    # Note
        网络层的第一层必须指定输入数据的形状，既在层的构造函数
        中添加`input_dim`参数（比如Dense）或者添加`input_shape`
        参数（比如Convolution2D层）。

    # Example
        ```python
            # TODO
        ```
    """

    def __init__(self, layers=None):
        self.layers = []  # 模型成员：网络层
        self.trainable_params = []  # 模型的所有可训练参数
        self.grads = []  # 模型的所有可训练参数的梯度

        self.objective = None  # 目标函数，即损失函数
        self.optimizer = None  # 优化器
        self.trainer = SequentialTrainer()  # 训练器，负责管理模型的训练过程

        if layers:
            for layer in layers:
                self.add(layer)

    def add(self, layer):
        """添加一个网络层作为网络的最后一层。

        # Arguments:
            layer: 网络层对象。

        """
        if len(self.layers) > 0:
            self.layers[-1].latter_layer = layer
            layer.previous_layer = self.layers[-1]
        self.layers.append(layer)
        # 调用网络层对象的build方法来初始化网络层对象的参数
        layer.build()
        self.trainable_params.append(layer.trainable_params)

    def pop(self):
        """将网络的最后一层网络层移除掉。"""
        if not self.layers:
            raise Exception('There are no layers in the model.')

        self.layers.pop()
        self.layers[-1].latter_layer = None
        self.trainable_params.pop()

    def set_objective(self, objective):
        """设置目标函数/损失函数类型。

        # Arguments:
            objective: 字符串，表示目标函数的类型，比如'mse'或者
            'categorical_crossentropy'等等。

        # Note:
            参数是str类型。
        """
        self.objective = objectives.get(objective)

    def set_optimizer(self, optimizer):
        """设置优化器。

        # Arguments:
            optimizers: 优化器对象，比如`SGD(lr=0.1)`，

        # Note:
            参数不是str类型。
        """
        self.optimizer = optimizer

    def summary(self):
        """返回一个描述了模型构造信息的字符串。"""
        info = '================ model summary ==============================\n'
        for layer in self.layers:
            info += layer.summary() + '\n'
        info += 'loss: ' + str(self.objective.__name__) + '\n'
        info += 'optimizer: ' + str(self.optimizer.__class__.__name__) + '\n'
        info += '============================================================='
        return info

    def forward(self, input_data, mode='test'):
        """前向计算过程。

        # Arguments:
            input_data: 输入数据，numpy.ndarray对象，
            遵循NM格式或者NCHW格式。

            mode: 前向计算的模式。
                - `train`: 前向计算过程会保留中间计算结果。
                - `test`: 前向计算过程不会保留中间计算结果。

        """
        data = input_data
        for layer in self.layers:
            layer.mode = mode
            data = layer.forward(data)
        return data

    def predict(self, input_data):
        """根据输入数据计算预测结果。

        # Arguments:
            input_data: 输入数据，numpy.ndarray对象，
            遵循NM格式或者NCHW格式。
        """
        output_data = self.forward(input_data)
        return output_data.argmax(axis=1)

    def backward(self, d_output_data):
        """后向计算过程，依次将误差从后向前传播，并依次调用
        每一层的backward方法来求目标函数objective对每一层
        的参数的梯度。
        每一层的参数的梯度保存在每一层的grads列表中。

        # Arguments:
            d_output_data: 目标函数objective对于网络输出
            output_data的梯度。前缀`d_`表示这是一个梯度值。

        # Note:
            该过程不会更新参数。
        """
        d_data = d_output_data
        for i in xrange(len(self.layers)-1, -1, -1):
            d_data = self.layers[i].backward(d_data)
        return d_data

    def evaluate_accuracy(self, x, y, evaluate_batch_size=None):
        """根据数据集x及其标签y，评估模型的分类准确率。

        对于大型数据集支持批量计算。

        # Arguments:
            x: 数据集，遵循NM格式或者NCHW格式。

            y: one_hot化的标签，n*d维numpy.ndarray对象，
            一个维度代表一个类别。

            evaluate_batch_size: 每次批量计算的数据个数，如果不指定
            则表示不使用批量计算，而是直接将所有数据一次性喂进网络中。

        # Note:
            如果数据集个数比较多、数据维度比较大或者模型复杂度比较高
            的时候，建议选择批量计算以防止爆内存。

            另外该函数不支持y的维度d为1的情况。
        """
        d = y.shape[1]
        assert d != 1, '数据集的类别数量不能为1！'

        n = x.shape[0]
        idx = range(n)

        if not evaluate_batch_size:
            evaluate_batch_size = n

        # 批量计算的次数（向上取整）
        batch_times = np.ceil(n * 1.0 / evaluate_batch_size).astype(np.int32)
        k = 0.  # 准确预测的数据个数

        for i in xrange(batch_times):
            batch_range = idx[i * evaluate_batch_size:(i + 1) * evaluate_batch_size]
            batch_x = x[batch_range]
            batch_y = y[batch_range]
            batch_n = batch_x.shape[0]

            y_pred = self.predict(batch_x)
            y_real = np_utils.to_categorical(batch_y)  # 将one hot数组转换为离散类别值
            k += np.ones(batch_n)[y_pred == y_real].sum()

        acc = k / n
        return acc

    def evaluate_loss(self, x, y, evaluate_batch_size=None):
        """根据数据集x及其标签y，评估模型的损失函数值。

        对于大型数据集支持批量计算。

        # Arguments:
            x: 数据集，遵循NM格式或者NCHW格式。

            y: 标签，n*d维numpy.ndarray对象。
            当m = 1时表示y是连续值，当前问题是回归问题；
            当m > 1时表示y是one hot化的类别标签。

            evaluate_batch_size: 每次批量计算的数据个数，如果不指定
            则表示不使用批量计算，而是直接将所有数据一次性喂进网络中。

        # Note:
            如果数据集个数比较多、数据维度比较大或者模型复杂度比较高
            的时候，建议选择批量计算以防止爆内存。
        """
        n = x.shape[0]
        idx = range(n)

        if not evaluate_batch_size:
            evaluate_batch_size = n

        # 批量计算的次数（向上取整）
        batch_times = np.ceil(n * 1.0 / evaluate_batch_size).astype(np.int32)  # 除法，向上取整
        loss = 0.

        for i in xrange(batch_times):
            batch_range = idx[i * evaluate_batch_size:(i + 1) * evaluate_batch_size]
            batch_x = x[batch_range]
            batch_y = y[batch_range]
            batch_n = batch_x.shape[0]

            output = self.forward(batch_x)
            loss += self.objective.loss(batch_y, output) * batch_n

        loss /= n
        return loss

    def evaluate_accuracy_and_loss(self, x, y, evaluate_batch_size=None):
        """根据数据集x及其标签y，评估模型的分类准确率以及损失函数值。对于大型数据集
        支持批量计算。

        因为评估模型准确率以及求损失函数值存在重复的计算过程，因此该函数将两个过程
        合并，以减少重复计算。

        # Arguments:
            x: 数据集，遵循NM格式或者NCHW格式。

            y: one_hot化的标签，n*d维numpy.ndarray对象，
            一个维度代表一个类别。

            evaluate_batch_size: 每次批量计算的数据个数，如果不指定
            则表示不使用批量计算，而是直接将所有数据一次性喂进网络中。

        # Note:
            如果数据集个数比较多、数据维度比较大或者模型复杂度比较高
            的时候，建议选择批量计算以防止爆内存。

            另外该函数不支持y的维度d为1的情况。
        """
        n = x.shape[0]
        idx = range(n)

        if not evaluate_batch_size:
            evaluate_batch_size = n

        batch_times = np.ceil(n * 1.0 / evaluate_batch_size).astype(np.int32)
        loss = 0.
        k = 0.  # 正确预测的数据个数

        for i in xrange(batch_times):
            batch_range = idx[i * evaluate_batch_size:(i + 1) * evaluate_batch_size]
            batch_x = x[batch_range]
            batch_y = y[batch_range]
            batch_n = batch_x.shape[0]

            output = self.forward(batch_x)
            loss += self.objective.loss(batch_y, output) * batch_n

            y_pred = output.argmax(axis=1)
            y_real = np_utils.to_categorical(batch_y)
            k += np.ones(batch_n)[y_pred == y_real].sum()

        acc = k / n
        loss /= n
        return acc, loss


    def _check_(self):
        """check the model before training."""
        if self.layers[-1].layer_type == 'softmax' and self.objective != 'categorical_crossentropy':
            raise Exception('Error objective function: when using softmax as the output layer, '
                            'the objective of modle must be categorical_crossentropy!')

        if self.layers[-1].layer_type != 'softmax' and self.objective == 'categorical_crossentropy':
            raise Exception('Error objective function: when using categorical_crossentropy as '
                            'the objective, the output layer of model must be softmax!')

    def train(self, training_data, nb_epochs, mini_batch_size=1, verbose=1, log_epoch=1,
              validation_data=None, shuffle=True, evaluate_batch_size=100, **kwargs):
        # TODO
        """训练模型。

        # Arguments:
            training_data:

        # Note:

        # Example:

            训练模型的方法：
            ```python
                # 下面的两种方法是等价的。

                # 方法一
                model = Sequential()
                model.train(...)

                # 方法二
                model = Sequential()
                trainer = SequentialTrainer()
                trainer.train(model, ...)

            ```
        """
        # check the model before training
        self._check_()
        self.trainer.train(self,
                           training_data,
                           nb_epochs,
                           mini_batch_size,
                           verbose,
                           log_epoch=log_epoch,
                           validation_data=validation_data,
                           shuffle=shuffle,
                           evaluate_batch_size=evaluate_batch_size)

    def plot_prediction(self, data):
        self.trainer.plot_prediction(self, data)

    def plot_training_iteration(self):
        self.trainer.plot_training_iteration() class SGD(Optimizer):

    def __init__(self, lr=0.01, momentum=0.0, decay=0.0):
        """随机梯度下降法，支持动量参数，支持学习衰减率"""
        super(SGD, self).__init__()
        self.lr = lr
        self.momentum = momentum
        self.decay = decay

        # 当前迭代期的学习率
        self.cur_lr = lr

        # 保存上次迭代期的参数改变量（速度）
        self.pre_velocities = []
        # 保存本次迭代期的参数该变量（速度）
        self.now_velocities = []

        # 当前迭代期数
        self.iterations = 0

    def update(self, params, grads):
        """
        # Arguments:
            params: list对象，params中的一个成员是一个由网络层的可训练
            参数组成的list对象，
            即`params=[[layer 1 params], [layer 2 params], ...]`

            grads:
        """
        # 如果是第一轮，则初始化“上次参数改变量”为0
        if self.iterations == 0:
            self.pre_velocities = [0] * len(params)

        # 由于有些层不包含参数，因此其可训练参数是一个空列表([])，
        # 所以通过将二维列表转换为一维列表来去除掉空列表，并方便后续计算
        params = [param for layer_param in params for param in layer_param]
        grads = [grad for layer_grad in grads for grad in layer_grad]

        # 开始更新模型参数，同时保存“本次参数改变量”
        for param, grad, pre_velocity in zip(params, grads, self.pre_velocities):
            now_velocity = - self.cur_lr * grad + pre_velocity * self.momentum
            param += now_velocity
            self.now_velocities.append(now_velocity)

        self.next_iteration()

    def next_iteration(self):
        """更新优化器的超参数"""
        self.iterations += 1
        self.cur_lr = self.lr * 1.0 / (1.0 + self.decay * self.iterations)
        self.pre_velocities = self.now_velocities
        self.now_velocities = [] def visualize_convolution_kernel(conv_layer, title='convolution layer visualizing'):
    N, _, _, _ = conv_layer.w.shape def split_training_data(data, nb_validation=None, split_ratio=None):
    x, y = data
    n = np.shape(x)[0]

    if nb_validation:
        if nb_validation >= n:
            raise Exception('Error: nb_validation is bigger than x.shape[0]!')
        else:
            idx = n - nb_validation
    else:
        if split_ratio:
            idx = int(n * split_ratio)
        else:
            raise Exception('Error: nb_validation and split_ratio can not '
                            'be None both!')
    training_x = x[:idx]
    training_y = y[:idx]
    validation_x = x[idx:]
    validation_y = y[idx:]
    training_data = (training_x, training_y)
    validation_data = (validation_x, validation_y)
    return training_data, validation_data
from hamaa.datasets.datasets import load_mnist_data
from hamaa.layers import Dense, Activation, Convolution2D, Flatten, MeanPooling2D
from hamaa.models import Sequential
from hamaa.optimizers import SGD
from hamaa.utils import vis_utils
from hamaa.utils.np_utils import split_training_data
# encoding: utf-8
"""
@author: monitor1379 
@contact: yy4f5da2@hotmail.com
@site: www.monitor1379.com

@version: 1.0
@license: Apache Licence
@file: example4_mnist_cnn.py
@time: 2016/10/20 23:49

Build a convolution neural network on MNIST.
"""



print 'loading MNIST dataset...'
training_data, test_data = load_mnist_data(nb_training=6000, nb_test=10000, preprocess=True, flatten=False)
training_data, validation_data = split_training_data(training_data, nb_validation=500)

print 'training_data:', training_data[0].shape
print 'validation_data:', validation_data[0].shape
print 'test_data:', test_data[0].shape

model = Sequential()
model.add(Convolution2D(nb_kernel=5, kernel_height=5, kernel_width=5, activation='relu', input_shape=(1, 28, 28)))
model.add(MeanPooling2D(pooling_size=(2, 2)))
model.add(Convolution2D(nb_kernel=5, kernel_height=5, kernel_width=5, activation='relu'))
model.add(MeanPooling2D(pooling_size=(2, 2)))
model.add(Flatten())
model.add(Dense(output_dim=200, init='glorot_normal'))
model.add(Activation('sigmoid'))
model.add(Dense(output_dim=10, init='glorot_normal'))
model.add(Activation('softmax'))

model.set_objective('categorical_crossentropy')
model.set_optimizer(SGD(lr=0.002, momentum=0.9))

print model.summary()

model.train(training_data=training_data,
            nb_epochs=20,
            mini_batch_size=64,
            verbose=2,
            validation_data=validation_data,
            log_epoch=1)

print model.evaluate_accuracy(test_data[0], test_data[1], evaluate_batch_size=64)

# visualize the kernels of convolution layer
vis_utils.visualize_convolution_kernel(model.layers[0], title='layer 0')

vis_utils.visualize_convolution_kernel(model.layers[2], title='layer 2')