class ConvolutionalAutoencoder(object):
    """A demo of a DEEP convAE. It is two parts:
    Encoder: a deep convolutional network.
    Decoder: a deep transposed convolutional network."""
    def __init__(self, input, encoder_config, decoder_config, crop_size,
                 cost_type=CostType.MeanSquared, rnd=None,
                 corruption_level=0., l1_reg=0., l2_reg=0., reg_bias=False):
        """crope_size = [Din, hight_img, width_img]"""
        self.encoder_config = encoder_config
        self.decoder_config = decoder_config
        self.catched_params = []
        self.cost_type = cost_type
        self.corruption_level = corruption_level
        self.x = input
        self.input = input
        self.params = []
        self.layers = []
        if rnd is None:
            rnd = np.random.RandomState(1231)
        self.theano_rng = RandomStreams(rnd.randint(2 ** 30))
        # Encoder part
        self.layers.append(
            DeepConvolutionLayer(input=self.x,
                                 layers=encoder_config,
                                 crop_size=crop_size))
        self.encoder = self.layers[0]
        self.params.extend(self.layers[0].params)
        code = self.layers[0].output
        output_features_dim = self.layers[0].output_features_dim
        # Decoder
        self.layers.append(
            DeepConvolutionLayer(input=code,
                                 layers=decoder_config,
                                 crop_size=output_features_dim))
        self.decoder = self.layers[1]
        self.output = self.layers[1].output
        self.params.extend(self.layers[1].params)
        self.L1 = 0
        self.L2 = 0
        self.L1_reg = l1_reg
        self.L2_reg = l2_reg
        for param in self.params:
            if ("w" in param.name) or ("W" in param.name):
                if l1_reg != 0.:
                    self.L1 += abs(param).sum()
                if l2_reg != 0.:
                    self.L2 += (param**2).sum()
            elif ("b" in param.name) or ("B" in param.name):
                if l1_reg != 0. and reg_bias:
                    self.L1 += abs(param).sum()
                if l2_reg != 0. and reg_bias:
                    self.L2 += (param**2).sum()
        if self.L1 != 0.:
            self.L1 *= sharedX_value(self.L1_reg, "l1_reg")
            print "Performing l1 reg. Lambda=", l1_reg
        if self.L2 != 0.:
            self.L2 *= sharedX_value(self.L2_reg, "l2_reg")
            print "Performing l2 reg. Lambda=", l2_reg

    def catch_params(self):
        self.catched_params = []
        for param in self.params:
            self.catched_params.append(param.get_value())

    def encode(self, x_in=None, train=False):
        if x_in is None:
            x_in = self.x
        self.encoder.x = x_in
        return self.encoder.output

    def decode(self, h):
        self.decoder.x = h
        return self.decoder.output

    def get_reconstructed_image(self, data):
        h = self.encode(x_in=data)
        x_rec = self.decode(h)
        return x_rec

    def get_rec_cost(self, x_rec, eyes=False):
        """
        Returns the reconstruction cost.
        """
        if self.cost_type == CostType.MeanSquared:
            return T.mean(((self.x.flatten(2) - x_rec)**2).sum(axis=1))
        elif self.cost_type == CostType.CrossEntropy:
            return T.mean(
                (T.nnet.binary_crossentropy(x_rec, self.x.flatten(2))).mean(axis=1))

    def get_reconstruction_error(self, x_in=None):
        if x_in is None:
            x_in = self.x
        h = self.encode(x_in, train=False)
        x_rec = self.decode(h)
        x_rec_faltten = x_rec.flatten(2)
        if self.cost_type == CostType.MeanSquared:
            return T.mean(((self.x.flatten(2) - x_rec_faltten)**2).sum(axis=1))
        elif self.cost_type == CostType.CrossEntropy:
            return T.mean(
                (T.nnet.binary_crossentropy(x_rec_faltten, self.x.flatten(2))).mean(axis=1))

    def get_train_cost(self, x_in=None):
        if x_in is None:
            if self.corruption_level != 0.:
                x_in = self.theano_rng.binomial(
                    x_in.shape, n=1, p=1-self.corruption_level,
                    dtype=theano.config.floatX) * self.x
            else:
                x_in = self.x
        h = self.encode(x_in)
        x_rec = self.decode(h)
        x_rec_faltten = x_rec.flatten(2)
        cost = self.get_rec_cost(x_rec_faltten)
        if self.L1_reg != 0.:
            cost += self.L1_reg * self.L1
        if self.L2_reg != 0.:
            cost += self.L2_reg * self.L2
        return (cost, h, x_rec)

    def save_params(self, weights_file, catched=False):
        """Save the model's parameters."""
        f_dump = open(weights_file, "w")
        params_vls = []
        if catched:
            if self.catched_params != []:
                params_vls = self.catched_params
            else:
                raise ValueError(
                    "You asked to save catched params," +
                    "but you didn't catch any!!!!!!!")
        else:
            for param in self.params:
                params_vls.append(param.get_value())
        pkl.dump(params_vls, f_dump, protocol=pkl.HIGHEST_PROTOCOL)
        f_dump.close()

    def set_params_vals(self, weights_file):
        """Set the values of the parameters."""
        with open(weights_file, 'r') as f:
            params_vls = pkl.load(f)
            for param, val in zip(self.params, params_vls):
                param.set_value(val)

    def switch_to_catched_params(self):
        """Set the model params to the catched ones."""
        if self.catched_params != []:
            for param, val in zip(self.params, self.catched_params):
                param.set_value(val) class Autoencoder(object):
    def __init__(
            self,
            input,
            nvis,
            nhid=None,
            nvis_dec=None,
            nhid_dec=None,
            rnd=None,
            bhid=None,
            cost_type=CostType.MeanSquared,
            momentum=1,
            num_pieces=1,
            L2_reg=-1,
            L1_reg=-1,
            sparse_initialize=False,
            nonlinearity=NonLinearity.TANH,
            bvis=None,
            tied_weights=True,
            reverse=False):
    def catch_params(self):
    def nonlinearity_fn(self, d_in=None, recons=False):
    def get_non_linearity_fn(self):
    def encode(self, x_in=None, center=True):
    def encode_linear(self, x_in=None):
    def decode(self, h):
    def get_rec_cost(self, x_rec, eyes=False):
    def get_rec_cost_face(self, x_rec):
    def kl_divergence(self, p, p_hat):
    def sparsity_penality(self, h, sparsity_level=0.05, sparse_reg=1e-3,
                          batch_size=-1):
    def act_grads(self, inputs):
    def jacobian_h_x(self, inputs):
    def compute_jacobian_h_x(self, inputs):
    def sample_one_step(self, x, sigma):
    def sample_scan(self, x, sigma, n_steps, samples):
    def sample_old(self, x, sigma, n_steps):
    def get_sgd_updates(self, learning_rate, lr_scaler=1.0, batch_size=1,
                        sparsity_level=-1, sparse_reg=-1, x_in=None):
    def get_train_cost(self, batch_size=1, sparsity_level=-1, sparse_reg=-1,
                       x_in=None, face=False):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def fit(self,
            data=None,
            learning_rate=0.1,
            batch_size=100,
            n_epochs=20,
            lr_scalar=0.998,
            weights_file="out/ae_weights_mnist.npy"):
        J = theano.gradient.jacobian(h, inputs) def split_data_to_minibatchs_eval(data, batch_size, sharpe=False):
    """Split a dataset to minibatchs to:
        1. Control the case where the size of the set is not divided by the
            btahc_size.
        2. Allows to share batch by batch when the size of data is too big to
            fit in the GPU shared memory.

    data: dictionnary with two keys:
        x: x (numpy.ndarray) for the supervised data
            may contains rows with only numpy.nan as values.
        y: y (numpy.ndarray) for the superrvised data
            may contains rows with only numpy.nan as values.
    batch_size: the size of the batch

    Returns:
        list_minibatchs: list
            a list of dic of mini-batchs, each one has the 2 keys: "x", "y".
    """
    nbr_samples = data['x'].shape[0]
    if batch_size > nbr_samples:
        batch_size = nbr_samples

    nbr_batchs = int(nbr_samples / batch_size)
    list_minibatchs = []
    for index in xrange(nbr_batchs):
        x_train = data['x'][index * batch_size:(index+1) * batch_size]
        y_train = data['y'][index * batch_size:(index+1) * batch_size]
        list_minibatchs.append({'x': x_train.astype(theano.config.floatX),
                                'y': y_train.astype(theano.config.floatX)})
    if not sharpe:
        # in case some samples are left
        if (nbr_batchs * batch_size) < nbr_samples:
            x_train = data['x'][(index+1) * batch_size:]
            y_train = data['y'][(index+1) * batch_size:]
            list_minibatchs.append({'x': x_train.astype(theano.config.floatX),
                                    'y': y_train.astype(theano.config.floatX)})

    return list_minibatchs def split_data_to_minibatchs_embed(data, batch_size, share=False, borrow=True,
                                   sharpe=False):
    """Split a dataset to minibatchs to:
        1. Control the case where the size of the set is not divided by the
            btahc_size.
        2. Allows to share batch by batch when the size of data is too big to
            fit in the GPU shared memory.

    data: dictionnary with two keys:
        x: x (numpy.ndarray) for the supervised data
            may contains rows with only numpy.nan as values.
        y: y (numpy.ndarray) for the superrvised data
            may contains rows with only numpy.nan as values.
    batch_size: the size of the batch
    sharpe: Boolean. If True, ignore the rest of data with size less than
        batch size.

    Returns:
        list_minibatchs: list
            a list of dic of mini-batchs, each one has the 3 keys: "in",
            "out", "sup".
    """
    nbr_samples = data['x'].shape[0]
    if batch_size > nbr_samples:
        batch_size = nbr_samples

    nbr_batchs = int(nbr_samples / batch_size)
    list_minibatchs = []
    for index in xrange(nbr_batchs):
        x_train = data['x'][index * batch_size:(index+1) * batch_size]
        y_train = data['y'][index * batch_size:(index+1) * batch_size]
        c_batch_in, c_batch_out, c_batch_sup = split_mini_batch(
            x_train.astype(theano.config.floatX),
            y_train.astype(theano.config.floatX), share=share, borrow=borrow)

        list_minibatchs.append(
            {'in': c_batch_in, 'out': c_batch_out, 'sup': c_batch_sup})
    if not sharpe:
        # in case some samples are left
        if (nbr_batchs * batch_size) < nbr_samples:
            x_train = data['x'][(index+1) * batch_size:]
            y_train = data['y'][(index+1) * batch_size:]
            c_batch_in, c_batch_out, c_batch_sup = split_mini_batch(
                x_train.astype(theano.config.floatX),
                y_train.astype(theano.config.floatX), share=share,
                borrow=borrow)
            list_minibatchs.append(
                {'in': c_batch_in, 'out': c_batch_out, 'sup': c_batch_sup})

    return list_minibatchs def plot_all_x_xhat(X, Xhat, nbr, path):
    if nbr > X.shape[0]:
        nbr = X.shape[0]
    for i in range(nbr):
        x, x_hat = X[i], Xhat[i]
        x = x.reshape((x.shape[1], x.shape[2]))
        x_hat = x_hat.reshape((x_hat.shape[1], x_hat.shape[2]))
        f = plot_x_x_yhat(x, x_hat)
        f.savefig(path + "/" + str(i) + ".png", bbox_inches='tight') class FaceDataset(object):
    def __init__(self):
        pass

    def create_ft_extractor(self, type_mod, weights_path):
        """Extract the features from x using a convnet model."""
#        model = convnet(type_mod, weights_path=weights_path, heatmap=False,
#                        W_regularizer=None,
#                        activity_regularizer=None,
#                        dense=False)
#        sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
#        model.compile(optimizer=sgd, loss="mse")
#        print "Summary:", model.summary()
        model = None

        return model

    def debug_plot_over_img(self, img, x, y, bb_d, bb_gt):
        """Plot the landmarks over the image with the bbox."""
        plt.close("all")
        fig = plt.figure()  # , figsize=(15, 10.8), dpi=200
        ax = plt.Axes(fig, [0., 0., 1., 1.])
        ax.set_axis_off()
        ax.imshow(img, aspect="auto", cmap='Greys_r')
        ax.scatter(x, y, s=10, color='r')
        rect1 = patches.Rectangle(
            (bb_d[0], bb_d[1]), bb_d[2]-bb_d[0], bb_d[3]-bb_d[1],
            linewidth=1, edgecolor='r', facecolor='none')
        ax.add_patch(rect1)
        rect2 = patches.Rectangle(
            (bb_gt[0], bb_gt[1]), bb_gt[2]-bb_gt[0], bb_gt[3]-bb_gt[1],
            linewidth=1, edgecolor='b', facecolor='none')
        ax.add_patch(rect2)
        fig.add_axes(ax)

        return fig

    def plot_over_img(self, img, x, y, x_pr, y_pr, bb_gt):
        """Plot the landmarks over the image with the bbox."""
        plt.close("all")
        fig = plt.figure(frameon=False)  # , figsize=(15, 10.8), dpi=200
        ax = plt.Axes(fig, [0., 0., 1., 1.])
        ax.set_axis_off()
        ax.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), aspect="auto")
        ax.scatter(x, y, s=10, color='r')
        ax.scatter(x_pr, y_pr, s=10, color='g')
        rect = patches.Rectangle(
            (bb_gt[0], bb_gt[1]), bb_gt[2]-bb_gt[0], bb_gt[3]-bb_gt[1],
            linewidth=1, edgecolor='b', facecolor='none')
        ax.add_patch(rect)
        fig.add_axes(ax)

        return fig

    def plot_over_img_seg(self, img, x, y, x_pr, y_pr, bb_gt, tag_oc=None):
        """Plot the landmarks over the image with the bbox."""
        plt.close("all")
        fig = plt.figure(frameon=False)  # , figsize=(15, 10.8), dpi=200
        ax = plt.Axes(fig, [0., 0., 1., 1.])
        ax.set_axis_off()
        bb_gt = [int(xx) for xx in bb_gt]
        hight, width = bb_gt[3]-bb_gt[1], bb_gt[2]-bb_gt[0]
        if tag_oc is None:
            img_oc = copy.deepcopy(img)
        elif tag_oc is "left":
            img_oc = copy.deepcopy(img)
            p = int((20/50.) * width)  # we took only 20 pixels from 50.
            img_oc[bb_gt[1]:bb_gt[3],
                   bb_gt[0]:bb_gt[0]+p, :] = np.uint8(255/2.)
        elif tag_oc is "right":
            img_oc = copy.deepcopy(img)
            p = int((20/50.) * width)  # we took only 20 pixels from 50.
            img_oc[bb_gt[1]:bb_gt[3],
                   bb_gt[2]-p:bb_gt[2], :] = np.uint8(255/2.)
        elif tag_oc is "up":
            img_oc = copy.deepcopy(img)
            p = int((20/50.) * hight)  # we took only 20 pixels from 50.
            img_oc[bb_gt[1]:bb_gt[1]+p,
                   bb_gt[0]:bb_gt[2], :] = np.uint8(255/2.)
        elif tag_oc is "down":
            img_oc = copy.deepcopy(img)
            p = int((20/50.) * hight)  # we took only 20 pixels from 50.
            img_oc[bb_gt[3]-p:bb_gt[3],
                   bb_gt[0]:bb_gt[2], :] = np.uint8(255/2.)
        elif tag_oc is "middle":
            img_oc = copy.deepcopy(img)
            p1 = int((15/50.) * hight)  # we took only from 15 pixels from 50.
            p2 = int((35/50.) * hight)  # we took only to 35 pixels from 50.
            img_oc[bb_gt[1]+p1:bb_gt[1]+p2,
                   bb_gt[0]:bb_gt[2], :] = np.uint8(255/2.)
        ax.imshow(cv2.cvtColor(img_oc, cv2.COLOR_BGR2RGB), aspect="auto")
        for i in xrange(68):
            ax.plot([x[i], x_pr[i]], [y[i], y_pr[i]], '-r')

        fig.add_axes(ax)

        return fig

    def debug_draw_set(self, path_f, fd_out):
        if not os.path.exists(fd_out):
            os.makedirs(fd_out)
        with open(path_f, 'r') as f:
            stuff = pkl.load(f)
        for el in stuff:
            img_name = el["img_name"]
            print img_name
            img_gray = el["img_gray"]
            bb_d = el["bb_detector"]
            bb_gt = el["bb_ground_truth"]
            x = el["annox"]
            y = el["annoy"]
            fig = self.debug_plot_over_img(img_gray, x, y, bb_d, bb_gt)
            fig.savefig(
                fd_out+"/"+img_name, bbox_inches='tight', pad_inches=0,
                frameon=False, cmap=cm.Greys_r)
            del fig

    def rescale_bb(self, img_shape, bb):
        return bb
        # How much to inrease the booox
        inc_w, inc_h = 0., 0.
        x1, y1, x2, y2 = bb
        w_bb = y2 - y1
        h_bb = x2 - x1
        c_x, c_y = x1 + h_bb/2, y1 + w_bb/2
        w_bb = w_bb + inc_w * w_bb
        h_bb = h_bb + inc_h * h_bb
        h_bb = h_bb + h_bb/2
        x1, y1 = c_x - h_bb/2, c_y - w_bb/2
        x2, y2 = c_x + h_bb/2, c_y + w_bb/2
        if x1 < 0:
            x1 = 0
        if y1 < 0:
            y1 = 0
        if x2 >= img_shape[1]:
            x2 = img_shape[1] - 1
        if y2 >= img_shape[0]:
            y2 = img_shape[0] - 1
        return [x1, y1, x2, y2]

    def normalizr_img_color_cnn(self, img, color_mode):
        if color_mode == "bgr":
            img[:, :, [0, 1, 2]] = img[:, :, [2, 1, 0]]
        img[:, :, 0] -= 123.68
        img[:, :, 1] -= 116.779
        img[:, :, 2] -= 103.939
        img = img.transpose((2, 0, 1))

        return img

    def sample_no_ag(self, img, bb, annox, annoy, w, h,
                     color_mode=None, ft_extract=None, get_original=False):
        [x1, y1, x2, y2] = [int(e) for e in bb]
        w_bb, h_bb = y2 - y1, x2 - x1
        if ft_extract is not None:
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            img = self.normalizr_img_color_cnn(img, color_mode)

        if ft_extract is None:
            cropped = img[y1:y2+1, x1:x2+1] / 255.
        else:
            cropped = img[:, y1:y2+1, x1:x2+1] / 255.
            cropped = cropped.transpose((1, 2, 0))

        im = cv2.resize(cropped, (w, h), interpolation=cv2.INTER_LINEAR)
        if ft_extract is not None:
            im = im.transpose((2, 0, 1))
            input_cnn = np.empty((1, 3, w, h), dtype=theano.config.floatX)
            input_cnn[0] = im
            im = ft_extract.predict(input_cnn)

        #fig = self.debug_plot_over_img(cropped, annox, annoy, bb, bb)
        #fig.savefig("2.png",  bbox_inches='tight')
        #raw_input("wait")

        # Normlaize in new box
        if not get_original:
            annox = (annox - x1)
            annoy = (annoy - y1)

            rx = h / float(h_bb)
            ry = w / float(w_bb)
            annox *= rx
            annoy *= ry
            annox /= float(h)  # in [0, 1] (no scaling!!!!)
            annoy /= float(w)  # in [0, 1] (no scaling!!!!)
            annox -= 0.5  # normalize into [-1, 1]
            annoy -= 0.5  # normalize into [-1, 1]

        # Rormlaize in new box END.
        # Or rescale only into the original box
#        annox = (annox - x1) / float(h_bb)  # in [0, 1] (no scaling!!!!)
#        annoy = (annoy - y1) / float(w_bb)  # in [0, 1] (no scaling!!!!)
        #fig = self.debug_plot_over_img(cropped, annox, annoy, bb, bb)
        #fig.savefig("3.png",  bbox_inches='tight')
        #raw_input("wait 2")
        return im, annox, annoy

    def inter_ocular_dist(self, annox, annoy):
        """Calculate the Euclidean  distance between the outer corners of the
            eyes."""
        return np.sqrt((annox[37] - annox[46])**2 + (annoy[37] - annoy[46])**2)

    def sample_from_list_to_test(self, l_samples, w, h):
        x, y = self.sample_from_list(l_samples, w, h, d=0, get_original=True)
        l_infos = []
        i = 0
        for e in l_samples:
            l_infos.append({
                "im_rgb": e["img_rgb"],
                "im_gray": e["img_gray"],
                "im_name": e["img_name"],
                "annox": y[i, :68],
                "annoy": y[i, 68:],
                "bb_gt": e["bb_ground_truth"],
                "d_eyes": self.inter_ocular_dist(e["annox"], e["annoy"])})
            i += 1

        return x, y, l_infos

    def calculate_face_errors(self, pred, l_infos, w, h):
        l_nrmse = []
        out = []
        for i in range(pred.shape[0]):
            el = l_infos[i]
            img_rgb = el["im_rgb"]
            img_name = el["im_name"]
            annox, annoy = el["annox"], el["annoy"]
            d_eyes, bb_gt = el["d_eyes"], el["bb_gt"]
            x_pred, y_pred = pred[i, 0:68], pred[i, 68:]
            # Convert prediction
            bb_gt = self.rescale_bb(el["im_gray"].shape, bb_gt)
            [x1, y1, x2, y2] = [int(e) for e in bb_gt]
            w_bb, h_bb = y2 - y1, x2 - x1
            # Norlalize in the new box

            rx = h / float(h_bb)
            ry = w / float(w_bb)
            x_pred += 0.5
            y_pred += 0.5
            x_pred *= h
            y_pred *= w
            x_pred /= rx
            y_pred /= ry
            x_pred += x1
            y_pred += y1

            # Rescale only in the original box.
#            x_pred = x_pred * float(h_bb) + x1
#            y_pred = y_pred * float(w_bb) + y1
            tmpx = np.power(np.subtract(annox, x_pred), 2)
            tmpy = np.power(np.subtract(annoy, y_pred), 2)
            tmpxy = np.sum(np.power((tmpx + tmpy), 0.5))
            nrmse = tmpxy / (68. * d_eyes)
            l_nrmse.append(nrmse)
            fig = self.plot_over_img(img_rgb, annox, annoy, x_pred,
                                     y_pred, bb_gt)
            out.append({"fig": fig, "img_name": img_name, "nrmse": nrmse})

        return np.asarray(l_nrmse), out

    def calculate_face_errors_octave(self, pred, l_infos, w, h, seg=False,
                                     tags_oc=None):
        octave.addpath("../")
        l_nrmse = []
        out = []
        nfids = np.int(pred.shape[1]/2)
        n_ims = pred.shape[0]
        y_reshaped = np.zeros((nfids, 2, n_ims))
        model_output_reshaped = np.zeros((nfids, 2, n_ims))
        saved_pred = copy.deepcopy(pred)

        for i in range(pred.shape[0]):
            el = l_infos[i]
            img_rgb = el["im_rgb"]
            img_name = el["im_name"]
            annox, annoy = el["annox"], el["annoy"]
            d_eyes, bb_gt = el["d_eyes"], el["bb_gt"]
            x_pred, y_pred = pred[i, 0:68], pred[i, 68:]
            # Convert prediction
            bb_gt = self.rescale_bb(el["im_gray"].shape, bb_gt)
            [x1, y1, x2, y2] = [int(e) for e in bb_gt]
            w_bb, h_bb = y2 - y1, x2 - x1
            # Normalize in the new box

            rx = h / float(h_bb)
            ry = w / float(w_bb)

            x_pred += 0.5
            y_pred += 0.5
            x_pred *= h
            y_pred *= w
            x_pred /= rx
            y_pred /= ry
            x_pred += x1
            y_pred += y1
            y_reshaped[:, 0, i] = annox  # x
            y_reshaped[:, 1, i] = annoy  # y
            model_output_reshaped[:, 0, i] = x_pred  # x
            model_output_reshaped[:, 1, i] = y_pred  # x

        l_nrmse = octave.compute_error_ibug(y_reshaped, model_output_reshaped)
        for i in range(saved_pred.shape[0]):
            el = l_infos[i]
            img_rgb = el["im_rgb"]
            img_name = el["im_name"]
            annox, annoy = el["annox"], el["annoy"]
            d_eyes, bb_gt = el["d_eyes"], el["bb_gt"]
            x_pred, y_pred = saved_pred[i, 0:68], saved_pred[i, 68:]
            # Convert prediction
            bb_gt = self.rescale_bb(el["im_gray"].shape, bb_gt)
            [x1, y1, x2, y2] = [int(e) for e in bb_gt]
            w_bb, h_bb = y2 - y1, x2 - x1
            # Norlalize in the new box

            rx = h / float(h_bb)
            ry = w / float(w_bb)
            x_pred += 0.5
            y_pred += 0.5
            x_pred *= h
            y_pred *= w
            x_pred /= rx
            y_pred /= ry
            x_pred += x1
            y_pred += y1

            nrmse = l_nrmse[i]
            tag_oc = None
            if tags_oc is not None:
                tag_oc = tags_oc[i]
            if seg:
                fig = self.plot_over_img_seg(img_rgb, annox, annoy, x_pred,
                                             y_pred, bb_gt, tag_oc=tag_oc)
            else:
                fig = self.plot_over_img(img_rgb, annox, annoy, x_pred,
                                         y_pred, bb_gt)
            out.append({"fig": fig, "img_name": img_name, "nrmse": nrmse})

        return np.asarray(l_nrmse), out

    def calculate_auc(self, y, dx):
        """ calculate the Area Under the Curve using the average of the
            estimated areas by the two composites the Simpsons's and the
            trapezoidal rules.
        """
        AUC = (simps(y, dx=dx) + trapz(y, dx=dx)) / 2.
        return AUC

    def calculate_auc_cdf(self, l_nrmse, bx, dx):
        """Draw the CDF curve and calculate its AUC."""
        x = np.arange(0, bx, dx)
        nbr_x = x.size
        cdf = np.zeros(nbr_x)
        # CDF curve
        for i in xrange(nbr_x):
            cdf[i] = np.sum((l_nrmse <= x[i]) * 1.)
        cdf /= float(len(l_nrmse))  # Get the percentage
        cdf0_1 = np.sum((l_nrmse <= 0.1) * 1.) / float(len(l_nrmse))
        # AUC
        area = self.calculate_auc(cdf, dx)
        all_area = bx * 1.
        auc = 100. * area/all_area

        return cdf, cdf0_1, auc

    def plot_cdf(self, cdf, tag, cdf0_1, auc, bx, dx):
        """Plot the cdf curve"""
        plt.close("all")
        x = np.arange(0, bx, dx)
        fig, ax = plt.subplots(nrows=1, ncols=1)
        ax.plot(x, cdf, label="CDF")
        ax.grid(True)
        plt.xlabel("NRMSE")
        plt.ylabel("Data proportion")
        plt.legend(loc=4, prop={'size': 8}, fancybox=True, shadow=True)
        plt.title("CDF curve: " + tag + ". CDF0.1: " + str(prec2 % cdf0_1) +
                  " . AUC:" + str(prec2 % auc) + ".")
        return fig

    def plot_cdf_model_and_meansh(self, cdfs, tag, cdf0_1s, aucs, bx, dx):
        plt.close("all")
        x = np.arange(0, bx, dx)
        fig, ax = plt.subplots(nrows=1, ncols=1)
        ax.plot(x, cdfs[0], label="CDF model")
        ax.plot(x, cdfs[1], label="CDF mean shape")
        ax.grid(True)
        plt.xlabel("NRMSE")
        plt.ylabel("Data proportion")
        plt.legend(loc=4, prop={'size': 8}, fancybox=True, shadow=True)
        plt.title(
            "CDF curve: " + tag + ". Model: CDF0.1: " +
            str(prec2 % cdf0_1s[0]) + " . AUC:" + str(prec2 % aucs[0]) +
            ".\n" + ". MSh: CDF0.1: " +
            str(prec2 % cdf0_1s[1]) + " . AUC:" + str(prec2 % aucs[1]) + ".\n")
        return fig

    def sample_from_list(self, l_samples, w, h, d=10, color_mode=None,
                         ft_extract=None, get_original=False):
        x = []  # np.empty((1, w*h), dtype=theano.config.floatX)
        y = []  # np.empty((1, 68*2), dtype=theano.config.floatX)
        kk = 0
        for el in l_samples:
            im_gray = el["img_gray"]
            print "\r Image ", kk, "/", len(l_samples),
            sys.stdout.flush()
            kk += 1
            bb_gt = el["bb_ground_truth"]
            annox = el["annox"]
            annoy = el["annoy"]
            bb = self.rescale_bb(im_gray.shape, bb_gt)
            img_shape = im_gray.shape
            im_rgb = None
            if ft_extract is not None:
                im_rgb = el["img_rgb"].astype(np.float32)
            if d == 0:
                if ft_extract is None:
                    s, xs, ys = self.sample_no_ag(
                        im_gray, bb, annox, annoy, w, h,
                        get_original=get_original)
                else:
                    s, xs, ys = self.sample_no_ag(
                        im_rgb, bb, annox, annoy, w, h, color_mode, ft_extract,
                        get_original=get_original)
                x.append(s.flatten())
                y.append(xs.tolist() + ys.tolist())
            else:
                k = 0
                # First: Extract the original bbox
                if ft_extract is None:
                    s, xs, ys = self.sample_no_ag(
                        im_gray, bb, annox, annoy, w, h,
                        get_original=get_original)
                else:
                    s, xs, ys = self.sample_no_ag(
                        im_rgb, bb, annox, annoy, w, h, color_mode, ft_extract,
                        get_original=get_original)
                x.append(s.flatten())
                y.append(xs.tolist() + ys.tolist())
                # Second: do data augmentation
                for dx in range(-d, d+1, 4):
                    for dy in range(-d, d+1, 4):
                        [x1, y1, x2, y2] = bb
                        x1 += dx
                        y1 += dy
                        x2 += dx
                        y2 += dy
                        if x1 < 0:
                            x1 = 0
                        if y1 < 0:
                            y1 = 0
                        if x2 >= img_shape[1]:
                            x2 = img_shape[1] - 1
                        if y2 >= img_shape[0]:
                            y2 = img_shape[0] - 1
                        bb_new = [x1, y1, x2, y2]
                        if x2 - x1 < 50:
                            # print ">>>>>>>>>>>>>>>>>>> Skipped x", bb_new
                            continue
                        if y2 - y1 < 50:
                            # print ">>>>>>>>>>>>>>>>>>> Skipped y", bb_new
                            continue
#                        fig = self.debug_plot_over_img(im_gray, annox, annoy, bb_new, bb_gt)
#                        fig.savefig("wtf2.png")
                        if ft_extract is None:
                            s, xs, ys = self.sample_no_ag(
                                im_gray, bb_new, annox, annoy, w, h,
                                get_original=get_original)
                        else:
                            s, xs, ys = self.sample_no_ag(
                                im_rgb, bb_new, annox, annoy, w, h, color_mode,
                                ft_extract, get_original=get_original)
#                        sm.imsave("fd/" + str(k) + '.png', s)
#                        print k
#                        k += 1
                        # Add noise to the images
                        s = s.flatten()
                        n_to_prod = np.array(np.random.binomial(
                            n=1, p=1 - corruption_level,
                            size=s.size),
                            dtype=theano.config.floatX)
#                        sm.imsave("fd/" + str(k) + '.png',
#                                  np.multiply(s, n_to_prod).reshape((w, h)))
#                        sm.imsave("fd/" + str(k) + '_orig.png',
#                                  s.reshape((w, h)))
#                        k += 1
#                        raw_input("Halt")
                        x.append(np.multiply(s, n_to_prod))
                        y.append(xs.tolist() + ys.tolist())

        x = np.asarray(x, dtype=theano.config.floatX)
        x = self.normalize_x_e_wise(x)
        y = np.asarray(y, dtype=theano.config.floatX)
        return x, y

    def normalize_y_negative(self, y):
        """Normalize y intro [-1, 1]."""
        y = y - 0.5

        return y

    def normalize_x_e_wise(self, x):
        """Normalize each row to be in [-1, 1]."""
        for i in range(x.shape[0]):
            vec = x[i]
            min_vec, max_vec = np.min(vec), np.max(vec)
            if max_vec != min_vec:
                vec = (vec - min_vec)/(max_vec-min_vec)
            else:
                vec = vec - min_vec
            x[i] = vec

        return x

    def suffle_x_y(self, x, y):
        xh = x.shape[1]
        data = np.hstack((x, y))
        for i in range(10):
            np.random.shuffle(data)
        x = data[:, :xh]
        y = data[:, xh:]

        return x, y

    def get_data(self, l_samples, xy, x, y):
        xy_data = l_samples[:xy]
        x_data = l_samples[xy:xy+x]
        y_data = l_samples[xy+x:xy+x+y]
        return xy_data, x_data, y_data

    def turn_to_nan(self, x):
        for i in range(x.shape[0]):
            x[i, :] = np.float32(np.nan)

        return x

    def chunks(self, list_s, n):
        out = []
        for i in range(0, len(list_s), n):
            out.append(list_s[i:i+n])
        return out

    def split_data(self, list_samples, xys, xy, x, y, w, h, fd_out, d=10,
                   type_mod=None, weights_path=None, color_mode=None):
        """xys: number of samples (x,y) in package."""
        ft_extract = None
        if type_mod is not None:
            ft_extract = self.create_ft_extractor(type_mod, weights_path)
        xys = int(xys)
        xy_data, x_data, y_data = self.get_data(list_samples, xy, x, y)
        xy_chunks = self.chunks(xy_data, xys)
        for ch in xy_chunks:
            print len(ch)
        x_chunks, y_chunks = None, None
        if x != 0:
            x_chunks = self.chunks(x_data, xys)
        if y != 0:
            y_chunks = self.chunks(y_data, xys)
        if type_mod is None:
            type_mod = ""
        if x == 0:
            i = 0
            for ch in xy_chunks:
                print "\n Chunk ", i, "/", len(xy_chunks)
                xy_x, xy_y = self.sample_from_list(ch, w, h, d, color_mode,
                                                   ft_extract)
                xy_x, xy_y = self.suffle_x_y(xy_x, xy_y)
                out = {"x": xy_x, "y": xy_y}
                p_file = type_mod + "ch_tr_" + str(xy) + "_" + str(x) + "_" +\
                    str(y) + "_" + str(i) + ".pkl"
                with open(fd_out + "/" + p_file, 'w') as f:
                    pkl.dump(out, f, pkl.HIGHEST_PROTOCOL)
                del xy_x
                del xy_y
                del out
                i += 1
        else:
            for i in range(len(xy_chunks)):
                print "\n Chunk ", i, "/", len(xy_chunks)
                ch_xy = xy_chunks[i]
                xy_x, xy_y = self.sample_from_list(ch_xy, w, h, d, color_mode,
                                                   ft_extract)
                xy_x, xy_y = self.suffle_x_y(xy_x, xy_y)
                x_x, x_y, y_x, x_y = None, None, None, None
                if i < len(x_chunks):
                    ch_x = x_chunks[i]
                    x_x, x_y = self.sample_from_list(ch_x, w, h, d, color_mode,
                                                     ft_extract)
                    x_x, x_y = self.suffle_x_y(x_x, x_y)
                    x_y = self.turn_to_nan(x_y)
                    xy_x = np.vstack((xy_x, x_x))
                    xy_y = np.vstack((xy_y, x_y))
                    xy_x, xy_y = self.suffle_x_y(xy_x, xy_y)
                if i < len(y_chunks):
                    ch_y = y_chunks[i]
                    y_x, y_y = self.sample_from_list(ch_y, w, h, d, color_mode,
                                                     ft_extract)
                    y_x, y_y = self.suffle_x_y(y_x, y_y)
                    y_x = self.turn_to_nan(y_x)
                    xy_x = np.vstack((xy_x, y_x))
                    xy_y = np.vstack((xy_y, y_y))
                    xy_x, xy_y = self.suffle_x_y(xy_x, xy_y)
                out = {"x": xy_x, "y": xy_y}
                p_file = type_mod + "ch_tr_" + str(xy) + "_" + str(x) + "_" +\
                    str(y) + "_" + str(i) + ".pkl"
                with open(fd_out + "/" + p_file, 'w') as f:
                    pkl.dump(out, f, pkl.HIGHEST_PROTOCOL)
                del xy_x
                del xy_y
                del out
                del x_x
                del x_y
                del y_x
                del y_y

    def prapere_valid(self, list_samples, fd_out, w, h, d=10,
                      type_mod=None, weights_path=None, color_mode=None):
        ft_extract = None
        if type_mod is not None:
            ft_extract = self.create_ft_extractor(type_mod, weights_path)
        x, y = self.sample_from_list(list_samples, w, h, d, color_mode,
                                     ft_extract)
        if type_mod is None:
            type_mod = ""
        with open(fd_out + '/' + type_mod + "valid.pkl", 'w') as f:
            out = {"x": x, "y": y}
            pkl.dump(out, f, pkl.HIGHEST_PROTOCOL)
            del x
            del y
            del out

    def find_mean_shape(self, fd):
        y_mean = None
        nbr = 0.
        for file in glob.glob(fd + "/ch_*.pkl"):
            with open(file, 'r') as f:
                if y_mean is None:
                    stuff = pkl.load(f)
                    y_mean = np.sum(stuff["y"], axis=0)
                else:
                    y_mean += np.sum(pkl.load(f)["y"], axis=0)
                nbr += stuff["y"].shape[0]
        y_mean /= nbr
        print y_mean.shape
        with open(fd + "mean_shape.pkl", 'w') as f:
            pkl.dump(y_mean, f, protocol=pkl.HIGHEST_PROTOCOL) def plot_fig(values, title, x_str, y_str, path, best_iter, std_vals=None):
    """Plot some values.
    Input:
         values: list or numpy.ndarray of values to plot (y)
         title: string; the title of the plot.
         x_str: string; the name of the x axis.
         y_str: string; the name of the y axis.
         path: string; path where to save the figure.
         best_iter: integer. The epoch of the best iteration.
         std_val: List or numpy.ndarray of standad deviation values that
             corresponds to each value in 'values'.
    """
    floating = 6
    prec = "%." + str(floating) + "f"

    if best_iter >= 0:
        if isinstance(values, list):
            if best_iter >= len(values):
                best_iter = -1
        if isinstance(values, np.ndarray):
            if best_iter >= np.size:
                best_iter = -1

        v = str(prec % np.float(values[best_iter]))
    else:
        v = str(prec % np.float(values[-1]))
        best_iter = -1
    if best_iter == -1:
        best_iter = len(values)
    fig = plt.figure()
    plt.plot(
        values,
        label="lower val: " + v + " at " + str(best_iter) + " " +
        x_str)
    plt.xlabel(x_str)
    plt.ylabel(y_str)
    plt.title(title, fontsize=8)
    plt.legend(loc='upper right', fancybox=True, shadow=True, prop={'size': 8})
    plt.grid(True)
    fig.savefig(path, bbox_inches='tight')
    plt.close('all')
    del fig class ModelMLP(object):
class ModelCNN(object):
class StaticExponentialDecayWeightRate(object):
class StaticExponentialDecayWeightRateSingle(object):
class StaticAnnealedWeightRate(object):
class StaticAnnealedWeightRateSingle(object):
    def __init__(self, layers_infos, input, l1_reg=0., l2_reg=0., tag="",
                 reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, layers_infos, input, crop_size, l1_reg=0., l2_reg=0.,
                 tag="", reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None, corruption_level=0., rnd=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, down, init_vl, end_vl, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
def floatX(arr):
def sharedX_mtx(mtx, name=None, borrow=None, dtype=None):
def contains_nan(array):
def contains_inf(array):
def isfinite(array):
def get_cost(aes, l, eye=True):
def get_net_pure_cost(model, cost_type, eye=True):
def get_net_cost(model, cost_type, eye=True):
def prepare_updates(model, aes_in, aes_out, cost, cost_in, cost_out, cost_sup,
                    cost_code, learning_rate, l_sup, l_code,
                    updaters={"all": None, "in": None, "out": None},
                    in3D=False):
def get_cost_code(model, aes_out):
def theano_fns(model,
               aes_in, aes_out, l_in, l_out, l_sup, l_code,
               learning_rate, cost_type,
               updaters={"all": None, "in": None, "out": None},
               max_colm_norm=False, max_norm=15.0, eye=True, in3D=False,
               id_code=None, use_dice=False):
def get_eval_fn(model, in3D=False, use_dice=False):
def evaluate_model(list_minibatchs_vl, eval_fn):
def evaluate_model_3D_unsup(list_minibatchs_vl, eval_fn):
def plot_fig(values, title, x_str, y_str, path, best_iter, std_vals=None):
def plot_stats(train_stats, ax, fold_exp, tag):
def print_stats_train(train_stats, epoch, ext, mb):
def train_one_epoch(train_updates, eval_fn, list_minibatchs_tr,
                    l_in, l_out, l_sup, list_minibatchs_vl,
                    model, aes_in, aes_out, epoch, fold_exp,
                    train_stats, vl_error_begin, tag):
def train_one_epoch_chuncks(train_updates, eval_fn, l_ch_tr,
                            l_in, l_out, l_sup, l_code, list_minibatchs_vl,
                            model, aes_in, aes_out, epoch, fold_exp,
                            train_stats, vl_error_begin, tag, tr_batch_size,
                            stop=False):
def collect_stats_epoch(stats, train_stats):
def split_mini_batch(x_train, y_train, share=True, borrow=True):
def split_data_to_minibatchs_embed(data, batch_size, share=False, borrow=True,
                                   sharpe=False):
def split_data_to_minibatchs_eval(data, batch_size, sharpe=False):
def grab_train_data(x, y, nbr_sup, nbr_xx, nbr_yy):
def reshape_data(y, s):
def plot_x_y_yhat(x, y, y_hat, xsz, ysz, binz=False):
def plot_x_x_yhat(x, x_hat):
def plot_all_x_xhat(X, Xhat, nbr, path):
def plot_all_x_y_yhat(x, y, yhat, xsz, ysz, fd, binz=False): def evaluate_model_3D_unsup(list_minibatchs_vl, eval_fn):
    """Evalute the model over a set."""
    error, output, code = None, None, None
    for mn_vl in list_minibatchs_vl:
        x = theano.shared(
            mn_vl['x'], borrow=True).get_value(borrow=True)

        [error_mn, output_mn, code_mn] = eval_fn(x)
        if error is None:
            error = error_mn
            output = output_mn
            code = code_mn
        else:
            error = np.vstack((error, error_mn))
            output = np.vstack((output, output_mn))
            code = np.vstack((code, code_mn))

    return error, output, code class Momentum(LearningRule):
    """Implementation of the momentum as in the method described in section
    9 of [1]:'A Practical Guide to Training Restricted Boltzmann Machines',
    bu Geoffrey Hinton.(https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf)
    We implemented alos the formula presented in Imagenet paper:
    <ImageNet Classification with Deep Convolutional Neural Networks>,
    A.Krizhevsky et al. .
    More details in:
    [2]'On the importance of initialization and momentum in deep learning',
    I. Sutskever et al.
    [3]'Advances in optimizating recurrent networks', Y. Bengio et al.

    The model's parametes are updated such as:
    velocity_(t+1) := momentum * velocity_t -
        learning_rate * d cost / d param_t
    param_(t+1) := param_t + velocity_(t+1)

    Parameters:
        init_momentum: float
            Initial value of the momentum coefficient. It remains fisex unless
                used with 'MomentumAdjuster'.
        nesterov_momentum: boolean
            If True, uses the accelerated momentum technique described in [2,3]
        max_colm_norm: Boolean. The incoming weight vector corresponding to
            each hidden unit is constrained to have a maximum squared length of
            max_norm.
        max_norm: Float. The maximum norm.
    """
    def __init__(self, init_momentum, nesterov_momentum=False,
                 imagenet=False, imagenetDecay=5e-4, max_colm_norm=False,
                 max_norm=15.0):
        assert init_momentum >= 0., 'The initial momentum should be >=0.'
        assert init_momentum < 1., 'The initial momentum should be < 1.'

        self.momentum = sharedX_value(value=init_momentum, name="momentum",
                                      borrow=True)
        self.nesterov_momentum = nesterov_momentum
        self._first_time = True
        self.velocity = None  # tracks the velocity at the previous time
        self.imagenet = imagenet
        self.imagenetDecay = sharedX_value(value=imagenetDecay,
                                           name="imagenetDecay",
                                           borrow=True)
        self.max_colm_norm = max_colm_norm
        self.max_norm = max_norm

    def get_updates(self, learning_rate, params, grads, lr_scalers):
        """
        get the updates (params, and velocity)
        """
        # the initial velocity is zero.
        if self._first_time:
            self.velocity = [
                sharedX_mtx(
                    param.get_value() * 0.,
                    name='vel_'+param.name, borrow=True) for param in params]

        updates = []
        for (param, grad, vel, lr_sc) in zip(
                params, grads, self.velocity, lr_scalers):
            lr_scaled = learning_rate * lr_sc
            if self.imagenet:
                new_vel = self.momentum * vel -\
                    lr_scaled * self.imagenetDecay * param - lr_scaled * grad
            else:
                new_vel = self.momentum * vel - lr_scaled * grad

            updates.append((vel, new_vel))
            inc = new_vel
            # this is the equivalence of NAG in [3].3.5, eq [7].
            # It helps to avoid calculating the new grad(param+vel_(t-1)).
            # The only different from the paper is: momentum_(t)
            # which it's set to momentum_(t-1). If you develop the final inc,
            # you will find that it's equivalent to eq.[7] mentioned above.
            if self.nesterov_momentum:
                inc = self.momentum * new_vel - lr_scaled * grad

            new_param = param + inc
            if self.max_colm_norm and param.name in ["W", "w"]:
                new_param_final = norm_constraint(tensor_var=new_param,
                                                  max_norm=self.max_norm)
            else:
                new_param_final = new_param
            updates.append((param, new_param_final))

        # add the velocity updates to updates

        return updates class AdaDelta(LearningRule):
    """Implement the ADADELTA algorithm of [1] to update the parameters
    of the model.
    Parameters:
        decay: float
            Decay rate in [1].

    Caution: the parameter 'epsilon' in [1] is the learning rate.
    So It would be better to use a small learning rate
        (maybe fixed all the learning process [we will see]
    [1]:'AdaDelta: An Adaptive Learning Rate Method', Zeiler M. )
    """
    def __init__(self, decay=0.95, max_colm_norm=False, max_norm=15.0):
        assert decay >= 0., 'The decay parameter in ' + str(type(self)) +\
            ' must be >= 0.'
        assert decay < 1., 'The decay parameter in ' + str(type(self)) +\
            ' must be < 1.'
        self.decay = decay
        self._first_time = True
        self.mean_square_grad = None
        self.mean_squar_dx = None
        self.max_colm_norm = max_colm_norm
        self.max_norm = max_norm

    def get_updates(self, learning_rate, params, grads, lr_scalers):
        """Compute the AdaDelta updates of the model's parameters.

        param_t := param_(t-1) + AdaDelta_update_t
        """
        if self._first_time:
            self.mean_square_grad = [
                sharedX_mtx(
                    param.get_value() * 0.,
                    name='mean_square_grad_'+param.name,
                    borrow=True) for param in params]
            self.mean_squar_dx = [
                sharedX_mtx(
                    param.get_value() * 0.,
                    name='mean_square_dx_'+param.name,
                    borrow=True) for param in params]
            self._first_time = False

        updates = []
        for (param, grad, mean_square_grad, mean_squar_dx, lr_sc) in zip(
                params, grads, self.mean_square_grad, self.mean_squar_dx,
                lr_scalers):
            # Calculate the running average gradient: E[g^2]_t
            new_mean_square_grad = (
                self.decay * mean_square_grad + (1 - self.decay) * T.sqr(grad))

            # The update: delta_x_t
            lr_scaled = learning_rate * lr_sc
            epsilon = lr_scaled
            rms_dx_t_1 = T.sqrt(mean_squar_dx + epsilon)
            rms_grad_t = T.sqrt(new_mean_square_grad + epsilon)
            delta_x_t = - (rms_dx_t_1 / rms_grad_t) * grad
            # Compute: E[delta_x^2]_t
            new_mean_square_dx = (
                self.decay * mean_squar_dx +
                (1 - self.decay) * T.sqr(delta_x_t))

            # update the params
            new_param = param + delta_x_t
            # Send for the update
            updates.append((mean_square_grad, new_mean_square_grad))
            updates.append((mean_squar_dx, new_mean_square_dx))
            if self.max_colm_norm and param.name in ["W", "w"]:
                new_param_final = norm_constraint(tensor_var=new_param,
                                                  max_norm=self.max_norm)
            else:
                new_param_final = new_param
            updates.append((param, new_param_final))

        return updates
from sop_embed.da import ConvolutionalAutoencoder
from sop_embed.ae import CostType
from sop_embed.tools import split_data_to_minibatchs_eval
from sop_embed.tools import split_data_to_minibatchs_embed
from sop_embed.tools import plot_all_x_xhat
from sop_embed.facedataset import FaceDataset
from sop_embed.tools import plot_fig
from sop_embed.tools import sharedX_value
from sop_embed.tools import evaluate_model_3D_unsup
from sop_embed.learning_rule import Momentum
from sop_embed.learning_rule import AdaDelta
import theano.tensor as T
import datetime as DT
import os
import sys
import numpy as np
import cPickle as pkl
import theano
import inspect
import shutil
import yaml
id_deep_conv_ae = id_deep_conv_ae.replace("(", "")
id_deep_conv_ae = id_deep_conv_ae.replace(")", "")
print len(id_deep_conv_ae), id_deep_conv_ae
path_ini_params_deep_conv_ae = init_w_deep_conv_ae_path +\
    "deep_conv_ae_init_" + id_deep_conv_ae + ".pkl"
if not os.path.isfile(path_ini_params_deep_conv_ae):
    deep_conv_ae.save_params(path_ini_params_deep_conv_ae)
else:
    deep_conv_ae.set_params_vals(path_ini_params_deep_conv_ae)

with open(path_valid, 'r') as f:
        l_samples_vl = pkl.load(f)
        # convert to 3D
        nbr_xx = l_samples_vl["x"].shape[0]
        l_samples_vl["x"] = l_samples_vl["x"].reshape((nbr_xx, 1, h, w))
list_minibatchs_vl = split_data_to_minibatchs_eval(
    l_samples_vl, vl_batch_size)
max_epochs = int(1000)

lr_vl = 1e-4
lr = sharedX_value(lr_vl, name="lr")
optimizer = "momentum"
if optimizer == "adadelta":
    updater = AdaDelta(decay=0.95)
elif optimizer == "momentum":
    updater = Momentum(0.9, nesterov_momentum=True,
                       imagenet=False, imagenetDecay=5e-4,
                       max_colm_norm=False)
else:
    raise ValueError("Optimizer not recognized.")

# Prepare costs.
cost_train, _, _ = deep_conv_ae.get_train_cost()
# updates
grads = T.grad(cost_train, deep_conv_ae.params)
lr_sc_all = list([sharedX_value(1.) for i in xrange(len(deep_conv_ae.params))])
updates = updater.get_updates(lr, deep_conv_ae.params, grads, lr_sc_all)
x_train = T.tensor4('x_train')
sgd_update = theano.function(
    inputs=[x_train],
    outputs=[T.mean(cost_train)],
    updates=updates,
    givens={deep_conv_ae.x: x_train},
    on_unused_input="ignore")
# eval function
x_val = T.tensor4("x_vl")
recons_error = deep_conv_ae.get_reconstruction_error()
eval_fn = theano.function(
    inputs=[x_val],
    outputs=[recons_error, deep_conv_ae.output, deep_conv_ae.encoder.output],
    givens={deep_conv_ae.x: x_val})

# validate before start training
error_mn, _, _ = evaluate_model_3D_unsup(list_minibatchs_vl, eval_fn)
vl_error_begin = np.mean(error_mn)
print "Val before train:", vl_error_begin

shutil.copy(inspect.stack()[0][1], fold_exp)
l_ch_tr_vl = []
for ch in l_ch_tr:
    with open(ch, 'r') as f:
        l_samples = pkl.load(f)
        nbr_xx = l_samples["x"].shape[0]
        l_samples["x"] = l_samples["x"].reshape((nbr_xx, 1, h, w))
    l_ch_tr_vl.append(l_samples)

# Train
stop = False
i = 0
in_cost, error_vl = [], []
freq_vl = 10
count, best_count = 0, 0
catched_one = False
while i < max_epochs:
    for ch in l_ch_tr_vl:
        if isinstance(ch, str):
            print "\n Chunk:", ch
            with open(ch, 'r') as f:
                l_samples = pkl.load(f)
        else:
            l_samples = ch
        sharpe = False
        list_minibatchs_tr = split_data_to_minibatchs_embed(
            l_samples, tr_batch_size, share=False, sharpe=sharpe)
#        for ts in xrange(100):
#            shuffle(list_minibatchs_tr)
        for minibatch in list_minibatchs_tr:
            count += 1
            t0 = DT.datetime.now()
            in_cost_mb, out_cost_mb, all_cost_mb = 0., 0., 0.
            # share data ONLY when needed (save the GPU memory).
            d_in = minibatch['in']
            d_out = minibatch['out']
            d_sup = minibatch['sup']
            # Update unsupervised task in
            if (d_in['x'] is not None):
                xx = theano.shared(
                    d_in['x'], borrow=True).get_value(borrow=True)
                in_cost_mb = sgd_update(xx)[0]
                in_cost.append(in_cost_mb)
                print in_cost_mb
                del xx
            # Val.
            if (count % freq_vl == 0):
                error_mn, _, _ = evaluate_model_3D_unsup(list_minibatchs_vl,
                                                         eval_fn)
                error_vl.append(np.mean(error_mn))
                if len(error_vl) > 1:
                    min_val = np.min(error_vl[:-1])
                else:
                    min_val = vl_error_begin
                print "Min val:", min_val, " current val:", error_vl[-1]
                if error_vl[-1] < min_val:
                    deep_conv_ae.catch_params()
                    catched_one = True
                    best_count = count
    i += 1

# End training
# Plot training figures.

plot_fig(in_cost, "Train DeepConvAE cost (in).", "Minibatch", "Train cost",