class DenoisingAutoencoder(Autoencoder):
    """
    Basic class for the denoising autoencoder.
    """
    def __init__(self,
                 input,
                 nvis,
                 nhid,
                 rnd=None,
                 theano_rng=None,
                 bhid=None,
                 cost_type=CostType.MeanSquared,
                 momentum=1,
                 L1_reg=-1,
                 L2_reg=-1,
                 sparse_initialize=False,
                 nonlinearity=NonLinearity.TANH,
                 bvis=None,
                 tied_weights=True,
                 reverse=False,
                 corruption_level=0.):
        super(DenoisingAutoencoder, self).__init__(
            input=input,
            nvis=nvis,
            nhid=nhid,
            rnd=rnd,
            bhid=bhid,
            cost_type=cost_type,
            momentum=momentum,
            L1_reg=L1_reg,
            L2_reg=L2_reg,
            sparse_initialize=sparse_initialize,
            nonlinearity=nonlinearity,
            bvis=bvis,
            tied_weights=tied_weights,
            reverse=reverse)
        self.corruption_level = corruption_level

        if not theano_rng:
            theano_rng = RandomStreams(rnd.randint(2 ** 30))
        self.theano_rng = theano_rng

    # Overrite this function:
    def encode(self, x_in=None, center=True):
        print "Overriten ...."
        if x_in is None:
            if self.corruption_level != 0.:
                x_in = self.theano_rng.binomial(
                    self.x.shape, n=1, p=1-self.corruption_level,
                    dtype=theano.config.floatX) * self.x
            else:
                x_in = self.x

        act = self.nonlinearity_fn(T.dot(x_in, self.hidden.W) + self.hidden.b)
        if center:
            act = act - act.mean(0)
        return act

    def corrupt_input(self, in_data, corruption_level):
        return self.theano_rng.binomial(self.x.shape, n=1,
                                        p=1-corruption_level,
                                        dtype=theano.config.floatX) * self.x

    def get_reconstructed_images(self, data):
        h = self.encode(x_in=data)
        x_rec = self.decode(h)
        return x_rec

    def debug_grads(self, data):
        gfn = theano.function([self.x], self.gparams[0])
        print "gradients:"
        print gfn(data)
        print "params:"
        if not self.reverse:
            print self.hidden.W.get_value()
        else:
            print self.hidden.W_prime.get_value()

    def fit(self,
            data=None,
            learning_rate=0.1,
            learning_decay=None,
            batch_size=100,
            n_epochs=60,
            corruption_level=0.5,
            weights_file=None,
            sparsity_level=-1,
            sparse_reg=-1,
            shuffle_data=True,
            lr_scaler=1.0,
            recons_img_file="out/dae_reconstructed_pento.npy"):
        if data is None:
            raise Exception("Data can not be empty.")

        index = T.iscalar("index")
        data_shared = theano.shared(np.asarray(data.tolist(),
                                               dtype=theano.config.floatX))
        n_batches = data.shape[0] / batch_size

        corrupted_input = self.corrupt_input(data_shared, corruption_level)

        (cost, updates, h, x_rec) = self.get_sgd_updates(
            learning_rate, lr_scaler=lr_scaler, batch_size=batch_size,
            sparsity_level=sparsity_level, sparse_reg=sparse_reg,
            x_in=corrupted_input)

        train_ae = theano.function(
            [index], cost, updates=updates,
            givens={
                self.x: data_shared[index*batch_size: (index+1) * batch_size]})

        print "Start training DAE."
        ae_costs = []
        batch_index = 0
        for epoch in xrange(n_epochs):
            idxs = np.arange(n_batches)
            np.random.shuffle(idxs)
            print "Training at epoch %d" % epoch
            cost_one_epoch = []
            for batch_index in idxs:
                cost_one_epoch.append(train_ae(batch_index))
                if False:
                    print "Cost: ", ae_costs[-1]
                    self.debug_grads(
                        data_shared.get_value()
                        [batch_index * batch_size: (batch_index+1)*batch_size])
            print "Training at epohc %d, %f" % (epoch, np.mean(cost_one_epoch))
            ae_costs.append(np.mean(cost_one_epoch))

        if weights_file is not None:
            print "Saving the weights ..."
            self.save_params(weights_file)
        if recons_img_file is not None:
            print "Saving reconstructed images ..."
            x_rec = self.get_reconstructed_images(data_shared)
            np.save(recons_img_file, x_rec)
        return ae_costs class ModelMLP(object):
class ModelCNN(object):
class StaticExponentialDecayWeightRate(object):
class StaticExponentialDecayWeightRateSingle(object):
class StaticAnnealedWeightRate(object):
class StaticAnnealedWeightRateSingle(object):
    def __init__(self, layers_infos, input, l1_reg=0., l2_reg=0., tag="",
                 reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, layers_infos, input, crop_size, l1_reg=0., l2_reg=0.,
                 tag="", reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None, corruption_level=0., rnd=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, down, init_vl, end_vl, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
def floatX(arr):
def sharedX_mtx(mtx, name=None, borrow=None, dtype=None):
def contains_nan(array):
def contains_inf(array):
def isfinite(array):
def get_cost(aes, l, eye=True):
def get_net_pure_cost(model, cost_type, eye=True):
def get_net_cost(model, cost_type, eye=True):
def prepare_updates(model, aes_in, aes_out, cost, cost_in, cost_out, cost_sup,
                    cost_code, learning_rate, l_sup, l_code,
                    updaters={"all": None, "in": None, "out": None},
                    in3D=False):
def get_cost_code(model, aes_out):
def theano_fns(model,
               aes_in, aes_out, l_in, l_out, l_sup, l_code,
               learning_rate, cost_type,
               updaters={"all": None, "in": None, "out": None},
               max_colm_norm=False, max_norm=15.0, eye=True, in3D=False,
               id_code=None, use_dice=False):
def get_eval_fn(model, in3D=False, use_dice=False):
def evaluate_model(list_minibatchs_vl, eval_fn):
def evaluate_model_3D_unsup(list_minibatchs_vl, eval_fn):
def plot_fig(values, title, x_str, y_str, path, best_iter, std_vals=None):
def plot_stats(train_stats, ax, fold_exp, tag):
def print_stats_train(train_stats, epoch, ext, mb):
def train_one_epoch(train_updates, eval_fn, list_minibatchs_tr,
                    l_in, l_out, l_sup, list_minibatchs_vl,
                    model, aes_in, aes_out, epoch, fold_exp,
                    train_stats, vl_error_begin, tag):
def train_one_epoch_chuncks(train_updates, eval_fn, l_ch_tr,
                            l_in, l_out, l_sup, l_code, list_minibatchs_vl,
                            model, aes_in, aes_out, epoch, fold_exp,
                            train_stats, vl_error_begin, tag, tr_batch_size,
                            stop=False):
def collect_stats_epoch(stats, train_stats):
def split_mini_batch(x_train, y_train, share=True, borrow=True):
def split_data_to_minibatchs_embed(data, batch_size, share=False, borrow=True,
                                   sharpe=False):
def split_data_to_minibatchs_eval(data, batch_size, sharpe=False):
def grab_train_data(x, y, nbr_sup, nbr_xx, nbr_yy):
def reshape_data(y, s):
def plot_x_y_yhat(x, y, y_hat, xsz, ysz, binz=False):
def plot_x_x_yhat(x, x_hat):
def plot_all_x_xhat(X, Xhat, nbr, path):
def plot_all_x_y_yhat(x, y, yhat, xsz, ysz, fd, binz=False): class ModelMLP(object):
class ModelCNN(object):
class StaticExponentialDecayWeightRate(object):
class StaticExponentialDecayWeightRateSingle(object):
class StaticAnnealedWeightRate(object):
class StaticAnnealedWeightRateSingle(object):
    def __init__(self, layers_infos, input, l1_reg=0., l2_reg=0., tag="",
                 reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, layers_infos, input, crop_size, l1_reg=0., l2_reg=0.,
                 tag="", reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None, corruption_level=0., rnd=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, down, init_vl, end_vl, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
def floatX(arr):
def sharedX_mtx(mtx, name=None, borrow=None, dtype=None):
def contains_nan(array):
def contains_inf(array):
def isfinite(array):
def get_cost(aes, l, eye=True):
def get_net_pure_cost(model, cost_type, eye=True):
def get_net_cost(model, cost_type, eye=True):
def prepare_updates(model, aes_in, aes_out, cost, cost_in, cost_out, cost_sup,
                    cost_code, learning_rate, l_sup, l_code,
                    updaters={"all": None, "in": None, "out": None},
                    in3D=False):
def get_cost_code(model, aes_out):
def theano_fns(model,
               aes_in, aes_out, l_in, l_out, l_sup, l_code,
               learning_rate, cost_type,
               updaters={"all": None, "in": None, "out": None},
               max_colm_norm=False, max_norm=15.0, eye=True, in3D=False,
               id_code=None, use_dice=False):
def get_eval_fn(model, in3D=False, use_dice=False):
def evaluate_model(list_minibatchs_vl, eval_fn):
def evaluate_model_3D_unsup(list_minibatchs_vl, eval_fn):
def plot_fig(values, title, x_str, y_str, path, best_iter, std_vals=None):
def plot_stats(train_stats, ax, fold_exp, tag):
def print_stats_train(train_stats, epoch, ext, mb):
def train_one_epoch(train_updates, eval_fn, list_minibatchs_tr,
                    l_in, l_out, l_sup, list_minibatchs_vl,
                    model, aes_in, aes_out, epoch, fold_exp,
                    train_stats, vl_error_begin, tag):
def train_one_epoch_chuncks(train_updates, eval_fn, l_ch_tr,
                            l_in, l_out, l_sup, l_code, list_minibatchs_vl,
                            model, aes_in, aes_out, epoch, fold_exp,
                            train_stats, vl_error_begin, tag, tr_batch_size,
                            stop=False):
def collect_stats_epoch(stats, train_stats):
def split_mini_batch(x_train, y_train, share=True, borrow=True):
def split_data_to_minibatchs_embed(data, batch_size, share=False, borrow=True,
                                   sharpe=False):
def split_data_to_minibatchs_eval(data, batch_size, sharpe=False):
def grab_train_data(x, y, nbr_sup, nbr_xx, nbr_yy):
def reshape_data(y, s):
def plot_x_y_yhat(x, y, y_hat, xsz, ysz, binz=False):
def plot_x_x_yhat(x, x_hat):
def plot_all_x_xhat(X, Xhat, nbr, path):
def plot_all_x_y_yhat(x, y, yhat, xsz, ysz, fd, binz=False): class ModelMLP(object):
    def __init__(self, layers_infos, input, l1_reg=0., l2_reg=0., tag="",
                 reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None):
        self.layers_infos = [
            {"n_in": l["n_in"],
             "n_out": l["n_out"],
             "activation": l["activation"]} for l in layers_infos]
        if dropout is None:
            dropout = [0. for i in range(len(layers_infos))]
        self.tag = tag
        self.layers = []
        self.layers_dropout = []
        self.layer_dropout_code = None
        self.layer_code = None
        self.params_until_code = None
        self.dropout = dropout
        self.ft_extractor = ft_extractor
        self.state_train = theano.shared(0.)  # not used in this class.
        # catch the model's params in the memory WHITHOUT saving
        # them on disc because disc acces is so expensive on somme
        # servers.
        self.catched_params = []
        input_dropout = input
        if ft_extractor is None:
            self.x = input
        else:
            self.x = self.ft_extractor.inputs
            input_dropout = self.ft_extractor.output_dropout
        self.trg = T.fmatrix("y")
        self.params = []
        input_lr = input
        rng = np.random.RandomState(23355)
        input_lr_dp = DropoutIdentityHiddenLayer(rng, input_dropout,
                                                 dropout[0], False).output
        i = 0
        for layer, dout in zip(layers_infos, self.dropout[1:] + [0.]):
            self.layers_dropout.append(
                DropoutHiddenLayer(
                    rng=layer["rng"],
                    input=input_lr_dp,
                    n_in=layer["n_in"],
                    n_out=layer["n_out"],
                    dropout_rate=dout,
                    rescale=False,
                    W=layer["W"],
                    b=layer["b"],
                    activation=get_non_linearity_fn(layer["activation"])
                    )
            )
            self.layers.append(
                HiddenLayer(
                    input=input_lr,
                    n_in=layer["n_in"],
                    n_out=layer["n_out"],
                    W=self.layers_dropout[-1].W * (1 - dout),
                    b=self.layers_dropout[-1].b,
                    activation=get_non_linearity_fn(layer["activation"]),
                    rng=layer["rng"]
                    )
                )
            input_lr = self.layers[-1].output
            input_lr_dp = self.layers_dropout[-1].output
            self.params += self.layers_dropout[-1].params
            if id_code is not None:
                if i == id_code:
                    self.layer_dropout_code = self.layers_dropout[-1]
                    self.layer_code = self.layers[-1]
                    self.params_until_code = self.params[:]

        self.output = self.layers[-1].output
        self.output_dropout = self.layers_dropout[-1].output
        self.l1 = 0.
        self.l2 = 0.
        if self.ft_extractor is not None:
            self.params.extend(self.ft_extractor.params)
        nparams = np.sum(
            [p.get_value().flatten().shape[0] for p in self.params])

        print "model contains %i parameters" % nparams
        # dump params for debug
        todump = []
        for p in self.params:
            todump.append(p.get_value())
        with open("paramsmlp.pkl", "w") as f:
            pkl.dump(todump, f, protocol=pkl.HIGHEST_PROTOCOL)

#        with open("init_keras_param.pkl", 'r') as f:
#            keras_params = pkl.load(f)
#            print "initialized using keras params."
#            for param, param_vl in zip(self.params, keras_params):
#                param.set_value(param_vl)
        for l in self.layers:
            if l1_reg != 0.:
                self.l1 += abs(l.W).sum()
                if reg_bias:
                    self.l1 += abs(l.b).sum()
            if l2_reg != 0.:
                self.l2 += (l.W**2).sum()
                if reg_bias:
                    self.l2 += (l.b**2).sum()
        if self.l1 != 0.:
            self.l1 *= sharedX_value(l1_reg, "l1_reg")
        if self.l2 != 0.:
            self.l2 *= sharedX_value(l2_reg, "l2_reg")

    def catch_params(self):
        self.catched_params = [param.get_value() for param in self.params]

    def save_params(self, weights_file, catched=False):
        """Save the model's params."""
        with open(weights_file, "w") as f:
            if catched:
                if self.catched_params != []:
                    params_vl = self.catched_params
                else:
                    raise ValueError(
                        "You asked to save catched params," +
                        "but you didn't catch any!!!!!!!")
            else:
                params_vl = [param.get_value() for param in self.params]
            ft_extractor = False
            if self.ft_extractor is not None:
                ft_extractor = True
            stuff = {"layers_infos": self.layers_infos,
                     "params_vl": params_vl,
                     "tag": self.tag,
                     "dropout": self.dropout,
                     "ft_extractor": ft_extractor}
            pkl.dump(stuff, f, protocol=pkl.HIGHEST_PROTOCOL)

    def set_params_vals(self, weights_file):
        """Set the model's params."""
        with open(weights_file, "r") as f:
            stuff = pkl.load(f)
            layers_infos, params_vl = stuff["layers_infos"], stuff["params_vl"]
            # Stuff to check
            keys = ["n_in", "n_out", "activation"]
            assert all(
                [l1[k] == l2[k]
                    for k in keys
                    for (l1, l2) in zip(layers_infos, self.layers_infos)])
            for param, param_vl in zip(self.params, params_vl):
                param.set_value(param_vl) def train_one_epoch_chuncks(train_updates, eval_fn, l_ch_tr,
                            l_in, l_out, l_sup, l_code, list_minibatchs_vl,
                            model, aes_in, aes_out, epoch, fold_exp,
                            train_stats, vl_error_begin, tag, tr_batch_size,
                            stop=False):
    """ train the model's parameters for 1 epoch.
    """
    sgd_update_all = train_updates["all"]
    sgd_update_in = train_updates["in"]
    sgd_update_out = train_updates["out"]
    sgd_update_code = train_updates["code"]
    tr_pure_cost_fn = train_updates["cost_sup"]

    in_cost, out_cost, all_cost, tr_pure_cost, code_cost = [], [], [], [], []
    error_vl, error_tr = [], []
    nb_mb, best_epoch, best_mb = 0, 0, 0
    if epoch <= 1000:
        freq_vl = 2000
    elif epoch <= 1900 and epoch > 1000:
        freq_vl = 2000
    elif epoch > 1900:
        freq_vl = 1
    plot_freq = 20
    catchted_once = False

    for ch in l_ch_tr:
        if isinstance(ch, str):
            print "\n Chunk:", ch
            with open(ch, 'r') as f:
                l_samples = pkl.load(f)
        else:
            l_samples = ch
        sharpe = False
        if model.ft_extractor is not None:
            sharpe = True
        list_minibatchs_tr = split_data_to_minibatchs_embed(
            l_samples, tr_batch_size, share=False, sharpe=sharpe)
#        for ts in xrange(100):
#            shuffle(list_minibatchs_tr)
        for minibatch in list_minibatchs_tr:
            t0 = DT.datetime.now()
            in_cost_mb, out_cost_mb, all_cost_mb = 0., 0., 0.
            # share data ONLY when needed (save the GPU memory).
            d_in = minibatch['in']
            d_out = minibatch['out']
            d_sup = minibatch['sup']
            # Update unsupervised task in
            check_in = [l.get_value() != 0. for l in l_in]
            if (any(check_in)) and (d_in['x'] is not None) and\
                    (sgd_update_in is not None):
                xx = theano.shared(
                    d_in['x'], borrow=True).get_value(borrow=True)
                in_cost_mb = sgd_update_in(xx)[0]
                in_cost.append(in_cost_mb)
                train_stats["in_cost_mb"].append(in_cost_mb)
                del xx

            # Update unsupervised task out
            if (l_out[0].get_value() != 0.) and (d_out['y'] is not None) and\
                    (sgd_update_out is not None):
                xx = theano.shared(
                    d_out['y'], borrow=True).get_value(borrow=True)
                out_cost_mb = sgd_update_out(xx)[0]
                out_cost.append(out_cost_mb)
                train_stats["out_cost_mb"].append(out_cost_mb)
                del xx

            # Update the code:
            if (sgd_update_code is not None) and (l_code.get_value() != 0.)\
                    and (d_sup['x'] is not None) and\
                    (l_out[0].get_value() != 0.):
                x = theano.shared(
                    d_sup['x'], borrow=True).get_value(borrow=True)
                y = theano.shared(
                    d_sup['y'], borrow=True).get_value(borrow=True)
                code_cost.append(sgd_update_code(x, y)[0])
                train_stats["code_cost_mb"].append(code_cost[-1])
                del x
                del y

            # Update the supervised task.
            if (l_sup.get_value() != 0.) and (d_sup['x'] is not None):
                x = theano.shared(
                    d_sup['x'], borrow=True).get_value(borrow=True)
                y = theano.shared(
                    d_sup['y'], borrow=True).get_value(borrow=True)
                # track the pure tr cost: do it before the update
                tr_pure_cost.append(tr_pure_cost_fn(x, y)[0])
                train_stats["tr_pure_cost_mb"].append(tr_pure_cost[-1])
                # Update the params.
                all_cost_mb = sgd_update_all(x, y)[0]

                # Evaluate over train set
                error_mn, _ = eval_fn(x, y)
                error_tr.append(np.mean(error_mn))
                train_stats["error_tr_mb"].append(error_tr[-1])
                del x
                del y
            # Collect costs
            all_cost.append(in_cost_mb + out_cost_mb + all_cost_mb)
            train_stats["all_cost_mb"].append(all_cost[-1])

            # Validation
            # on somme server the access to the disc costs so much
            # time because the disc is not in the same machine
            # nor the same vlan!!!!!!!!!!!.
            if (nb_mb % freq_vl == 0):
                error_mn, _ = evaluate_model(list_minibatchs_vl, eval_fn)
                error_vl.append(np.mean(error_mn))
                train_stats["error_vl_mb"].append(error_vl[-1])
                # Pick the best model accordonly to the validation error of
                # the model.
                if len(train_stats["error_vl_mb"]) > 1:
                    min_vl = np.min(train_stats["error_vl_mb"][:-1])
                else:
                    min_vl = vl_error_begin
                print "vl error", error_vl[-1], epoch, "best:", min_vl,\
                    "min vl:", np.min(train_stats["error_vl_mb"])
                if error_vl[-1] < min_vl:
                    min_vl = error_vl[-1]
                    train_stats["best_epoch"] = epoch
                    train_stats["best_mb"] = len(
                        train_stats["error_vl_mb"]) - 1
                    model.catch_params()
                    catchted_once = True
                    if model.ft_extractor is not None:
                        model.ft_extractor.model.save(fold_exp + "/cnn.pkl")
#                    print "\t\t\tFound better model: SAVED. min vl:",\
#                        str(prec2 % min_vl)
                    if aes_in != []:
                        for l, k in zip(aes_in, range(len(aes_in))):
                            l.catch_params()

                    if aes_out != []:
                        for l, k in zip(aes_out, range(len(aes_out))):
                            l.catch_params()

            nb_mb += 1

            # Print train infos mb
#            print_stats_train(train_stats, epoch, "_mb", nb_mb-1)
    # save o disc best models.
    if (epoch % 100 == 0) or stop:
        if stop:
            print "Going to end the training, but before, I'm gonna go ahead "\
                " and save the model params."
        # Plot stats: mb
        plot_stats(train_stats, "mb", fold_exp, tag)
        if catchted_once:
#            if epoch > 1:
#                with open(fold_exp + "/vl_error.txt", "w") as f:
#                    f.write(repr(np.min(train_stats["error_vl"])))
            model.save_params(fold_exp + "/model.pkl", catched=True)
            if aes_in != []:
                for l, k in zip(aes_in, range(len(aes_in))):
                    if (epoch % 100 == 0):
                        l.save_params(
                            fold_exp + "/ae_in_" + str(k) + ".pkl",
                            catched=True)
                if aes_out != []:
                    for l, k in zip(aes_out, range(len(aes_out))):
                        if (epoch % 100 == 0):
                            l.save_params(
                                fold_exp + "/ae_out_" + str(k) + ".pkl",
                                catched=True)
#            print "Time mini-batch:", DT.datetime.now() - t0

#    # check if the params do not have bad values (inf, nan)
#    for param in self.params:
#        if not isfinite(param.get_value(borrow=True)):
#            raise Exception('Non finite values (inf or nan) in paramaters.')
#
#    get_info_params()
    # Stats
    stats = {"in_cost": in_cost,
             "out_cost": out_cost,
             "all_cost": all_cost,
             "tr_pure_cost": tr_pure_cost,
             "error_tr": error_tr,
             "error_vl": error_vl,
             "best_epoch": best_epoch,
             "best_mb": best_mb,
             "code_cost": code_cost}

    return stats def theano_fns(model,
               aes_in, aes_out, l_in, l_out, l_sup, l_code,
               learning_rate, cost_type,
               updaters={"all": None, "in": None, "out": None},
               max_colm_norm=False, max_norm=15.0, eye=True, in3D=False,
               id_code=None, use_dice=False):
    """
    """
    # code cost
    cost_code = None
    if id_code is not None:
        cost_code = get_cost_code(model, aes_out)
    # Get cost in
    cost_in = get_cost(aes_in, l_in, eye=False)
    # Get cost out
    cost_out = get_cost(aes_out, l_out, eye=eye)
    # Get cost network
    cost_net = get_net_cost(model, cost_type, eye=eye)
    # Get pure cost net
    cost_net_pure = get_net_pure_cost(model, cost_type, eye=eye)
    # Get the model error
    if use_dice:
        insec = T.sum(model.trg * model.output, axis=1)
        tmp = 1 - 2.0 * insec/(T.sum(model.trg, axis=1) + T.sum(model.output,
                               axis=1))
        error = T.mean(tmp)
    else:
        error = T.mean(T.mean((model.output - model.trg)**2, axis=1))

    # Total cost
    embed_cost = sharedX_value(0.)
    # Reviwer 1: Q1.
    if cost_in is not None:
        embed_cost += cost_in
    if cost_out is not None:
        embed_cost += cost_out
    if cost_code is not None:
        embed_cost += l_code * cost_code
    embed_cost += l_sup * cost_net
    # embed_cost = l_sup * cost_net

    # Train fn: Updates
    train_updates = prepare_updates(
        model, aes_in, aes_out, embed_cost, cost_in, cost_out, cost_net_pure,
        cost_code, learning_rate, l_sup, l_code,
        updaters=updaters, in3D=in3D)
    # Evaluation fn
    if in3D:
        x = T.tensor4('x')
    else:
        x = T.fmatrix("x")
    y = T.fmatrix("y")
    state_train = T.scalar(dtype=theano.config.floatX)

    output_fn_vl = [error, model.output]

    eval_fn = theano.function(
        [x, y, theano.In(state_train, value=0.)], output_fn_vl,
        givens={model.x: x,
                model.state_train: state_train,
                model.trg: y},
        on_unused_input='ignore')

    return train_updates, eval_fn class ModelMLP(object):
class ModelCNN(object):
class StaticExponentialDecayWeightRate(object):
class StaticExponentialDecayWeightRateSingle(object):
class StaticAnnealedWeightRate(object):
class StaticAnnealedWeightRateSingle(object):
    def __init__(self, layers_infos, input, l1_reg=0., l2_reg=0., tag="",
                 reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, layers_infos, input, crop_size, l1_reg=0., l2_reg=0.,
                 tag="", reg_bias=False, dropout=None, ft_extractor=None,
                 id_code=None, corruption_level=0., rnd=None):
    def catch_params(self):
    def save_params(self, weights_file, catched=False):
    def set_params_vals(self, weights_file):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, slop, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, anneal_start=0):
    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
    def get_current_weight_rate(self):
    def __init__(self, anneal_end, down, init_vl, end_vl, anneal_start=0):
    def __call__(self, w, epochs_seen):
    def get_current_weight_rate(self):
def floatX(arr):
def sharedX_mtx(mtx, name=None, borrow=None, dtype=None):
def contains_nan(array):
def contains_inf(array):
def isfinite(array):
def get_cost(aes, l, eye=True):
def get_net_pure_cost(model, cost_type, eye=True):
def get_net_cost(model, cost_type, eye=True):
def prepare_updates(model, aes_in, aes_out, cost, cost_in, cost_out, cost_sup,
                    cost_code, learning_rate, l_sup, l_code,
                    updaters={"all": None, "in": None, "out": None},
                    in3D=False):
def get_cost_code(model, aes_out):
def theano_fns(model,
               aes_in, aes_out, l_in, l_out, l_sup, l_code,
               learning_rate, cost_type,
               updaters={"all": None, "in": None, "out": None},
               max_colm_norm=False, max_norm=15.0, eye=True, in3D=False,
               id_code=None, use_dice=False):
def get_eval_fn(model, in3D=False, use_dice=False):
def evaluate_model(list_minibatchs_vl, eval_fn):
def evaluate_model_3D_unsup(list_minibatchs_vl, eval_fn):
def plot_fig(values, title, x_str, y_str, path, best_iter, std_vals=None):
def plot_stats(train_stats, ax, fold_exp, tag):
def print_stats_train(train_stats, epoch, ext, mb):
def train_one_epoch(train_updates, eval_fn, list_minibatchs_tr,
                    l_in, l_out, l_sup, list_minibatchs_vl,
                    model, aes_in, aes_out, epoch, fold_exp,
                    train_stats, vl_error_begin, tag):
def train_one_epoch_chuncks(train_updates, eval_fn, l_ch_tr,
                            l_in, l_out, l_sup, l_code, list_minibatchs_vl,
                            model, aes_in, aes_out, epoch, fold_exp,
                            train_stats, vl_error_begin, tag, tr_batch_size,
                            stop=False):
def collect_stats_epoch(stats, train_stats):
def split_mini_batch(x_train, y_train, share=True, borrow=True):
def split_data_to_minibatchs_embed(data, batch_size, share=False, borrow=True,
                                   sharpe=False):
def split_data_to_minibatchs_eval(data, batch_size, sharpe=False):
def grab_train_data(x, y, nbr_sup, nbr_xx, nbr_yy):
def reshape_data(y, s):
def plot_x_y_yhat(x, y, y_hat, xsz, ysz, binz=False):
def plot_x_x_yhat(x, x_hat):
def plot_all_x_xhat(X, Xhat, nbr, path):
def plot_all_x_y_yhat(x, y, yhat, xsz, ysz, fd, binz=False): def collect_stats_epoch(stats, train_stats):
    if stats["in_cost"] != []:
        train_stats["in_cost"].append(np.mean(stats["in_cost"]))
    if stats["out_cost"] != []:
        train_stats["out_cost"].append(np.mean(stats["out_cost"]))
    if stats["code_cost"] != []:
        train_stats["code_cost"].append(np.mean(stats["code_cost"]))
    train_stats["all_cost"].append(np.mean(stats["all_cost"]))
    train_stats["tr_pure_cost"].append(np.mean(stats["tr_pure_cost"]))
    train_stats["error_tr"].append(np.mean(stats["error_tr"]))
    train_stats["error_vl"].append(np.mean(stats["error_vl"]))

    return train_stats def plot_stats(train_stats, ax, fold_exp, tag):
    """Plot what happened during training.
    Input:
        ax: string. "mb": minbatch, "epoch": epoch.
    """
    if ax is "mb":
        fd = fold_exp + "/minibatches"
        extra1 = "_mb"
        extra2 = "minibatches"
        extra3 = train_stats["best_mb"]
    else:
        fd = fold_exp + "/epochs"
        extra1 = ""
        extra2 = "epochs"
        extra3 = train_stats["best_epoch"]
    if not os.path.exists(fd):
        os.makedirs(fd)

    if train_stats["all_cost" + extra1] != []:
        if train_stats["code_cost" + extra1] != []:
            plot_fig(train_stats["all_cost" + extra1],
                     "Train cost: sum(in+out+sup+code) [" + extra2 +
                     "]. Case: " + tag,
                     extra2, "cost: in+out+sup+code",
                     fd+"/train-cost-in+out+sup-code-" + extra2 + ".png",
                     extra3)
        else:
            plot_fig(train_stats["all_cost" + extra1],
                     "Train cost: sum(in+out+sup) [" + extra2 +
                     "]. Case: " + tag,
                     extra2, "cost: in+out+sup",
                     fd+"/train-cost-in+out+sup-" + extra2 + ".png",
                     extra3)

    if train_stats["code_cost" + extra1] != []:
        plot_fig(train_stats["code_cost" + extra1],
                 "Train cost: code [" + extra2 + "]. Case: " + tag,
                 extra2, "cost: code",
                 fd+"/train-cost-code-" + extra2 + ".png",
                 extra3)

    if train_stats["in_cost" + extra1] != []:
        plot_fig(train_stats["in_cost" + extra1],
                 "Train cost: in [" + extra2 + "]. Case: " + tag,
                 extra2, "cost: in",
                 fd+"/train-cost-in-" + extra2 + ".png",
                 extra3)

    if train_stats["out_cost" + extra1] != []:
        plot_fig(train_stats["out_cost" + extra1],
                 "Train cost: out [" + extra2 + "]. Case: " + tag,
                 extra2, "cost: out",
                 fd+"/train-cost-out-" + extra2 + ".png",
                 extra3)

    if train_stats["tr_pure_cost" + extra1] != []:
        plot_fig(train_stats["tr_pure_cost" + extra1],
                 "Train cost: pure sup [" + extra2 + "]. Case: " + tag,
                 extra2, "cost: pure sup",
                 fd+"/train-cost-pure-sup-" + extra2 + ".png",
                 extra3)

    if train_stats["error_tr" + extra1] != []:
        plot_fig(train_stats["error_tr" + extra1],
                 "Model train error: [" + extra2 + "]. Case: " + tag,
                 extra2, "Train error",
                 fd+"/train-error-" + extra2 + ".png",
                 extra3)

    if train_stats["error_vl" + extra1] != []:
        plot_fig(train_stats["error_vl" + extra1],
                 "Model valid error: [" + extra2 + "]. Case: " + tag,
                 extra2, "Valid error",
                 fd+"/valid-error-" + extra2 + ".png",
                 extra3) def split_data_to_minibatchs_eval(data, batch_size, sharpe=False):
    """Split a dataset to minibatchs to:
        1. Control the case where the size of the set is not divided by the
            btahc_size.
        2. Allows to share batch by batch when the size of data is too big to
            fit in the GPU shared memory.

    data: dictionnary with two keys:
        x: x (numpy.ndarray) for the supervised data
            may contains rows with only numpy.nan as values.
        y: y (numpy.ndarray) for the superrvised data
            may contains rows with only numpy.nan as values.
    batch_size: the size of the batch

    Returns:
        list_minibatchs: list
            a list of dic of mini-batchs, each one has the 2 keys: "x", "y".
    """
    nbr_samples = data['x'].shape[0]
    if batch_size > nbr_samples:
        batch_size = nbr_samples

    nbr_batchs = int(nbr_samples / batch_size)
    list_minibatchs = []
    for index in xrange(nbr_batchs):
        x_train = data['x'][index * batch_size:(index+1) * batch_size]
        y_train = data['y'][index * batch_size:(index+1) * batch_size]
        list_minibatchs.append({'x': x_train.astype(theano.config.floatX),
                                'y': y_train.astype(theano.config.floatX)})
    if not sharpe:
        # in case some samples are left
        if (nbr_batchs * batch_size) < nbr_samples:
            x_train = data['x'][(index+1) * batch_size:]
            y_train = data['y'][(index+1) * batch_size:]
            list_minibatchs.append({'x': x_train.astype(theano.config.floatX),
                                    'y': y_train.astype(theano.config.floatX)})

    return list_minibatchs def split_data_to_minibatchs_embed(data, batch_size, share=False, borrow=True,
                                   sharpe=False):
    """Split a dataset to minibatchs to:
        1. Control the case where the size of the set is not divided by the
            btahc_size.
        2. Allows to share batch by batch when the size of data is too big to
            fit in the GPU shared memory.

    data: dictionnary with two keys:
        x: x (numpy.ndarray) for the supervised data
            may contains rows with only numpy.nan as values.
        y: y (numpy.ndarray) for the superrvised data
            may contains rows with only numpy.nan as values.
    batch_size: the size of the batch
    sharpe: Boolean. If True, ignore the rest of data with size less than
        batch size.

    Returns:
        list_minibatchs: list
            a list of dic of mini-batchs, each one has the 3 keys: "in",
            "out", "sup".
    """
    nbr_samples = data['x'].shape[0]
    if batch_size > nbr_samples:
        batch_size = nbr_samples

    nbr_batchs = int(nbr_samples / batch_size)
    list_minibatchs = []
    for index in xrange(nbr_batchs):
        x_train = data['x'][index * batch_size:(index+1) * batch_size]
        y_train = data['y'][index * batch_size:(index+1) * batch_size]
        c_batch_in, c_batch_out, c_batch_sup = split_mini_batch(
            x_train.astype(theano.config.floatX),
            y_train.astype(theano.config.floatX), share=share, borrow=borrow)

        list_minibatchs.append(
            {'in': c_batch_in, 'out': c_batch_out, 'sup': c_batch_sup})
    if not sharpe:
        # in case some samples are left
        if (nbr_batchs * batch_size) < nbr_samples:
            x_train = data['x'][(index+1) * batch_size:]
            y_train = data['y'][(index+1) * batch_size:]
            c_batch_in, c_batch_out, c_batch_sup = split_mini_batch(
                x_train.astype(theano.config.floatX),
                y_train.astype(theano.config.floatX), share=share,
                borrow=borrow)
            list_minibatchs.append(
                {'in': c_batch_in, 'out': c_batch_out, 'sup': c_batch_sup})

    return list_minibatchs def evaluate_model(list_minibatchs_vl, eval_fn):
    """Evalute the model over a set."""
    error, output = None, None
    for mn_vl in list_minibatchs_vl:
        x = theano.shared(
            mn_vl['x'], borrow=True).get_value(borrow=True)
        y = theano.shared(
            mn_vl['y'], borrow=True).get_value(borrow=True)

        [error_mn, output_mn] = eval_fn(x, y)
        if error is None:
            error = error_mn
            output = output_mn
        else:
            error = np.vstack((error, error_mn))
            output = np.vstack((output, output_mn))
    return error, output class StaticAnnealedWeightRate(object):
    """
    A callback to adjust the cost weight rate.
    The weight is used to weight the PRE-TRAINING sub-costs in the embedded
    pre-training.

    The weight is annealed by: weight_sup = weight_sup + v,
    weight_in = 1 - v/2, weight_sup = 1 - v/2.
    v > 0.
    The annealing process starts from the epoch T0 = 0 (by defautl) and
    finishes at some epoch T.
    At the epoch T, the weight must be 0. Hince, v = weight/(T-T0).
    Parameters:
        anneal_end: int. The epoch when to end the annealing
        anneal_start: int (default 0). The epoch when to start annealing.
    """
    def __init__(self, anneal_end, anneal_start=0):
        self._initialized = False
        self._anneal_start = anneal_start
        self._anneal_end = anneal_end
        self.l_sup, self.l_in, self.l_out = None, None, None
        self.v = 0.

    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
        """Updates the weight rate according to the annealing schedule.
        l_sup: Theano shared variable.
            l_in: Theano shared variable.
            l_out: Theano shared variable.
            epochs_seen: Int. The number of seen epcohs.
            to_update: dict. indicate if to update l_in, l_out.
                {"l_in": True, "l_out": True}.
        """
        s_w = l_sup.get_value() + l_in.get_value() + l_out.get_value()
        assert -1 < 1e-3

        if not self._initialized:
            self.l_sup, self.l_in, self.l_out = l_sup, l_in, l_out

            distance = float(self._anneal_end - self._anneal_start)
            if distance == 0:
                self.v = 0
            else:
                self.v = float(1. - self.l_sup.get_value()) / distance

            self._initialized = True

        if (epochs_seen >= self._anneal_start) and \
                (epochs_seen <= self._anneal_end):
            self.l_sup.set_value(np.cast[theano.config.floatX](
                self.get_current_weight_rate()))
            if epochs_seen == self._anneal_end:
                self.l_sup.set_value(np.cast[theano.config.floatX](1.))

            val = 1. - self.l_sup.get_value()
            if to_update["l_in"] and to_update["l_out"]:
                self.l_in.set_value(np.cast[theano.config.floatX](val/2.))
                self.l_out.set_value(np.cast[theano.config.floatX](val/2.))
            elif to_update["l_in"] and not to_update["l_out"]:
                self.l_in.set_value(np.cast[theano.config.floatX](val))
            elif not to_update["l_in"] and to_update["l_out"]:
                self.l_out.set_value(np.cast[theano.config.floatX](val))

#        print l_sup.get_value(), l_in.get_value(), l_out.get_value()

        s_w = l_sup.get_value() + l_in.get_value() + l_out.get_value()
        assert s_w - 1 < 1e-3

    def get_current_weight_rate(self):
        """Calculate the current learning rate according to the annealing
        schedule.
        """
        return np.max([0., self.l_sup.get_value() + self.v]) class StaticExponentialDecayWeightRate(object):
    """
    A callback to adjust the cost weight rate.
    The weight is used to weight the PRE-TRAINING sub-costs in the embedded
    pre-training.

    The weights are updated by: weight_sup = 1 - v, weight_in = v/2
    weight_out = v/2.
    v > 0, v = exp(-epochs_seen/slope)
    Parameters:
        anneal_start: int (default 0). The epoch when to start annealing.
        slope: float. The slope of the exp.
    """
    def __init__(self, slop, anneal_start=0):
        self._initialized = False
        self._anneal_start = anneal_start
        self.l_sup, self.l_in, self.l_out = None, None, None
        self.slop = float(slop)
        self.epochs_seen = 0

    def __call__(self, l_sup, l_in, l_out, epochs_seen, to_update):
        """Updates the weight rate according to the exp schedule.
        Input:
            l_sup: Theano shared variable.
            l_in: Theano shared variable.
            l_out: Theano shared variable.
            epochs_seen: Int. The number of seen epcohs.
            to_update: dict. indicate if to update l_in, l_out.
                {"l_in": True, "l_out": True}.
        """
        s_w = l_sup.get_value() + l_in.get_value() + l_out.get_value()
        assert s_w - 1 < 1e-3
        self.epochs_seen = epochs_seen

        if not self._initialized:
            self.l_sup, self.l_in, self.l_out = l_sup, l_in, l_out
            self._initialized = True

        if (self.epochs_seen >= self._anneal_start):
            self.l_sup.set_value(np.cast[theano.config.floatX](
                self.get_current_weight_rate()))
            val = 1. - self.l_sup.get_value()
            if to_update["l_in"] and to_update["l_out"]:
                self.l_in.set_value(np.cast[theano.config.floatX](val/2.))
                self.l_out.set_value(np.cast[theano.config.floatX](val/2.))
            elif to_update["l_in"] and not to_update["l_out"]:
                self.l_in.set_value(np.cast[theano.config.floatX](val))
            elif not to_update["l_in"] and to_update["l_out"]:
                self.l_out.set_value(np.cast[theano.config.floatX](val))

        s_w = l_sup.get_value() + l_in.get_value() + l_out.get_value()
        assert s_w - 1 < 1e-3

    def get_current_weight_rate(self):
        """Calculate the current weight cost rate according to the a
        schedule.
        """
        return np.max([0., 1. - np.exp(float(-self.epochs_seen)/self.slop)]) class StaticExponentialDecayWeightRateSingle(object):
    """
    A callback to adjust the cost weight rate.
    The weight is used to weight the PRE-TRAINING sub-costs in the embedded
    pre-training.

    The weights are updated by: weight_sup = 1 - v, weight_in = v/2
    weight_out = v/2.
    v > 0, v = exp(-epochs_seen/slope)
    Parameters:
        anneal_start: int (default 0). The epoch when to start annealing.
        slope: float. The slope of the exp.
    """
    def __init__(self, slop, anneal_start=0):
        self._initialized = False
        self._anneal_start = anneal_start
        self.w = None
        self.slop = float(slop)
        self.epochs_seen = 0

    def __call__(self, w, epochs_seen):
        """Updates the weight rate according to the exp schedule.
        Input:
            w: Theano shared variable.
            epochs_seen: Int. The number of seen epcohs.
        """
        self.epochs_seen = epochs_seen

        if not self._initialized:
            self.w = w
            self._initialized = True

        if (self.epochs_seen >= self._anneal_start):
            self.w.set_value(np.cast[theano.config.floatX](
                self.get_current_weight_rate()))

    def get_current_weight_rate(self):
        """Calculate the current weight cost rate according to the a
        schedule.
        """
        return np.max([0., 1. - np.exp(float(-self.epochs_seen)/self.slop)]) class StaticAnnealedWeightRateSingle(object):
    """
    A callback to adjust the cost weight rate.
    The weight is used to weight the PRE-TRAINING sub-costs in the embedded
    pre-training.

    The weight is annealed by: weight_sup = weight_sup + v,
    weight_in = 1 - v/2, weight_sup = 1 - v/2.
    v > 0.
    The annealing process starts from the epoch T0 = 0 (by defautl) and
    finishes at some epoch T.
    At the epoch T, the weight must be 0. Hince, v = weight/(T-T0).
    Parameters:
        anneal_end: int. The epoch when to end the annealing
        anneal_start: int (default 0). The epoch when to start annealing.
    """
    def __init__(self, anneal_end, down, init_vl, end_vl, anneal_start=0):
        self._initialized = False
        self._anneal_start = anneal_start
        self._anneal_end = anneal_end
        self.w = None
        self.v = 0.
        self.epochs_seen = 0
        self.down = down
        self.init_vl = init_vl
        self.end_vl = end_vl

    def __call__(self, w, epochs_seen):
        """Updates the weight rate according to the annealing schedule.
        l_sup: Theano shared variable.
            l_in: Theano shared variable.
            l_out: Theano shared variable.
            epochs_seen: Int. The number of seen epcohs.
            to_update: dict. indicate if to update l_in, l_out.
                {"l_in": True, "l_out": True}.
        """
        self.epochs_seen = epochs_seen
        if not self._initialized:
            self.w = w

            distance = float(self._anneal_end - self._anneal_start)
            if distance == 0:
                self.v = 0
            else:
                if self.down:
                    self.v = float(self.end_vl - self.init_vl) / distance
                else:
                    self.v = float(self.init_vl - self.end_vl) / distance

            self._initialized = True

        if self.epochs_seen == self._anneal_start:
            self.w.set_value(np.cast[theano.config.floatX](self.init_vl))
        if (self.epochs_seen >= self._anneal_start) and \
                (self.epochs_seen <= self._anneal_end):
            self.w.set_value(np.cast[theano.config.floatX](
                self.get_current_weight_rate()))
            if epochs_seen == self._anneal_end:
                if self.down:
                    self.w.set_value(np.cast[theano.config.floatX](0.))
                else:
                    self.w.set_value(np.cast[theano.config.floatX](1.))

    def get_current_weight_rate(self):
        """Calculate the current learning rate according to the annealing
        schedule.
        """
        return np.max([0., self.w.get_value() + self.v]) def print_stats_train(train_stats, epoch, ext, mb):
    in_cost, out_cost, all_cost, tr_pure_cost = 0., 0., 0., 0.
    error_vl, error_tr, code_cost = 0., 0., 0.

    if train_stats["code_cost"+ext] != []:
        code_cost = str(prec2 % train_stats["code_cost"+ext][-1])
    if train_stats["in_cost"+ext] != []:
        in_cost = str(prec2 % train_stats["in_cost"+ext][-1])
    if train_stats["out_cost"+ext] != []:
        out_cost = str(prec2 % train_stats["out_cost"+ext][-1])
    if train_stats["tr_pure_cost"+ext] != []:
        tr_pure_cost = str(prec2 % train_stats["tr_pure_cost"+ext][-1])
    if train_stats["error_tr"+ext]:
        error_tr = str(prec2 % train_stats["error_tr"+ext][-1])
    all_cost = str(prec2 % train_stats["all_cost"+ext][-1])
    error_vl = str(prec2 % train_stats["error_vl"+ext][-1])
    min_vl_err = str(prec2 % min(train_stats["error_vl"+ext]))

    if ext is "":
        print "\r Epoch [", str(epoch), "] Train infos:",
    else:
        print "\r Epoch [", str(epoch), "] mb [", str(mb), "] Train infos:",
#    print "\t all cost:", all_cost, " \t tr pure cost:", tr_pure_cost
#    print "\t in cost:", in_cost, " \t out cost:", out_cost
#    print "\t error tr:", error_tr, " \t error vl:", error_vl, " min vl:",\
#        min_vl_err
#    print "\t all cost: %s ,\t tr pure cost: %s, \t in cost: %s,"\
#        "\t out cost: %s,\t error tr: %s, \t error vl: %s, \t min vl: %s"\
#        % (all_cost, tr_pure_cost, in_cost, out_cost, error_tr, error_vl,
#            min_vl_err)
    print "all cost:", all_cost, "\t tr pure cost:", tr_pure_cost,\
        "\t in cost:", in_cost, "\t  out cost:", out_cost, "\t error tr:",\
        error_tr, "\t code cost:", code_cost, "\t error vl:", error_vl,\
        "\t min vl:", min_vl_err,
    sys.stdout.flush() class AdaDelta(LearningRule):
    """Implement the ADADELTA algorithm of [1] to update the parameters
    of the model.
    Parameters:
        decay: float
            Decay rate in [1].

    Caution: the parameter 'epsilon' in [1] is the learning rate.
    So It would be better to use a small learning rate
        (maybe fixed all the learning process [we will see]
    [1]:'AdaDelta: An Adaptive Learning Rate Method', Zeiler M. )
    """
    def __init__(self, decay=0.95, max_colm_norm=False, max_norm=15.0):
        assert decay >= 0., 'The decay parameter in ' + str(type(self)) +\
            ' must be >= 0.'
        assert decay < 1., 'The decay parameter in ' + str(type(self)) +\
            ' must be < 1.'
        self.decay = decay
        self._first_time = True
        self.mean_square_grad = None
        self.mean_squar_dx = None
        self.max_colm_norm = max_colm_norm
        self.max_norm = max_norm

    def get_updates(self, learning_rate, params, grads, lr_scalers):
        """Compute the AdaDelta updates of the model's parameters.

        param_t := param_(t-1) + AdaDelta_update_t
        """
        if self._first_time:
            self.mean_square_grad = [
                sharedX_mtx(
                    param.get_value() * 0.,
                    name='mean_square_grad_'+param.name,
                    borrow=True) for param in params]
            self.mean_squar_dx = [
                sharedX_mtx(
                    param.get_value() * 0.,
                    name='mean_square_dx_'+param.name,
                    borrow=True) for param in params]
            self._first_time = False

        updates = []
        for (param, grad, mean_square_grad, mean_squar_dx, lr_sc) in zip(
                params, grads, self.mean_square_grad, self.mean_squar_dx,
                lr_scalers):
            # Calculate the running average gradient: E[g^2]_t
            new_mean_square_grad = (
                self.decay * mean_square_grad + (1 - self.decay) * T.sqr(grad))

            # The update: delta_x_t
            lr_scaled = learning_rate * lr_sc
            epsilon = lr_scaled
            rms_dx_t_1 = T.sqrt(mean_squar_dx + epsilon)
            rms_grad_t = T.sqrt(new_mean_square_grad + epsilon)
            delta_x_t = - (rms_dx_t_1 / rms_grad_t) * grad
            # Compute: E[delta_x^2]_t
            new_mean_square_dx = (
                self.decay * mean_squar_dx +
                (1 - self.decay) * T.sqr(delta_x_t))

            # update the params
            new_param = param + delta_x_t
            # Send for the update
            updates.append((mean_square_grad, new_mean_square_grad))
            updates.append((mean_squar_dx, new_mean_square_dx))
            if self.max_colm_norm and param.name in ["W", "w"]:
                new_param_final = norm_constraint(tensor_var=new_param,
                                                  max_norm=self.max_norm)
            else:
                new_param_final = new_param
            updates.append((param, new_param_final))

        return updates class Momentum(LearningRule):
    """Implementation of the momentum as in the method described in section
    9 of [1]:'A Practical Guide to Training Restricted Boltzmann Machines',
    bu Geoffrey Hinton.(https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf)
    We implemented alos the formula presented in Imagenet paper:
    <ImageNet Classification with Deep Convolutional Neural Networks>,
    A.Krizhevsky et al. .
    More details in:
    [2]'On the importance of initialization and momentum in deep learning',
    I. Sutskever et al.
    [3]'Advances in optimizating recurrent networks', Y. Bengio et al.

    The model's parametes are updated such as:
    velocity_(t+1) := momentum * velocity_t -
        learning_rate * d cost / d param_t
    param_(t+1) := param_t + velocity_(t+1)

    Parameters:
        init_momentum: float
            Initial value of the momentum coefficient. It remains fisex unless
                used with 'MomentumAdjuster'.
        nesterov_momentum: boolean
            If True, uses the accelerated momentum technique described in [2,3]
        max_colm_norm: Boolean. The incoming weight vector corresponding to
            each hidden unit is constrained to have a maximum squared length of
            max_norm.
        max_norm: Float. The maximum norm.
    """
    def __init__(self, init_momentum, nesterov_momentum=False,
                 imagenet=False, imagenetDecay=5e-4, max_colm_norm=False,
                 max_norm=15.0):
        assert init_momentum >= 0., 'The initial momentum should be >=0.'
        assert init_momentum < 1., 'The initial momentum should be < 1.'

        self.momentum = sharedX_value(value=init_momentum, name="momentum",
                                      borrow=True)
        self.nesterov_momentum = nesterov_momentum
        self._first_time = True
        self.velocity = None  # tracks the velocity at the previous time
        self.imagenet = imagenet
        self.imagenetDecay = sharedX_value(value=imagenetDecay,
                                           name="imagenetDecay",
                                           borrow=True)
        self.max_colm_norm = max_colm_norm
        self.max_norm = max_norm

    def get_updates(self, learning_rate, params, grads, lr_scalers):
        """
        get the updates (params, and velocity)
        """
        # the initial velocity is zero.
        if self._first_time:
            self.velocity = [
                sharedX_mtx(
                    param.get_value() * 0.,
                    name='vel_'+param.name, borrow=True) for param in params]

        updates = []
        for (param, grad, vel, lr_sc) in zip(
                params, grads, self.velocity, lr_scalers):
            lr_scaled = learning_rate * lr_sc
            if self.imagenet:
                new_vel = self.momentum * vel -\
                    lr_scaled * self.imagenetDecay * param - lr_scaled * grad
            else:
                new_vel = self.momentum * vel - lr_scaled * grad

            updates.append((vel, new_vel))
            inc = new_vel
            # this is the equivalence of NAG in [3].3.5, eq [7].
            # It helps to avoid calculating the new grad(param+vel_(t-1)).
            # The only different from the paper is: momentum_(t)
            # which it's set to momentum_(t-1). If you develop the final inc,
            # you will find that it's equivalent to eq.[7] mentioned above.
            if self.nesterov_momentum:
                inc = self.momentum * new_vel - lr_scaled * grad

            new_param = param + inc
            if self.max_colm_norm and param.name in ["W", "w"]:
                new_param_final = norm_constraint(tensor_var=new_param,
                                                  max_norm=self.max_norm)
            else:
                new_param_final = new_param
            updates.append((param, new_param_final))

        # add the velocity updates to updates

        return updates
import sys
import theano.tensor as T
import theano
import numpy as np
import cPickle as pkl
import datetime as DT
import os
import inspect
import shutil
from sop_embed.da import DenoisingAutoencoder
from sop_embed.tools import NonLinearity
from sop_embed.tools import CostType
from sop_embed.tools import ModelMLP
from sop_embed.tools import train_one_epoch_chuncks
from sop_embed.tools import theano_fns
from sop_embed.tools import sharedX_value
from sop_embed.tools import collect_stats_epoch
from sop_embed.tools import plot_stats
from sop_embed.tools import split_data_to_minibatchs_eval
from sop_embed.tools import split_data_to_minibatchs_embed
from sop_embed.tools import evaluate_model
from sop_embed.tools import StaticAnnealedWeightRate
from sop_embed.tools import StaticExponentialDecayWeightRate
from sop_embed.tools import StaticExponentialDecayWeightRateSingle
from sop_embed.tools import StaticAnnealedWeightRateSingle
from sop_embed.tools import print_stats_train
from sop_embed.learning_rule import AdaDelta
from sop_embed.learning_rule import Momentum
from random import shuffle
                                  rnd=rnd,
                                  nonlinearity=NonLinearity.SIGMOID,
                                  cost_type=CostType.MeanSquared,
                                  reverse=False,
                                  corruption_level=0.01)

    if not os.path.isfile(path_ini_params_l1):
        dae_l1.save_params(path_ini_params_l1)
    else:
        dae_l1.set_params_vals(path_ini_params_l1)

    # Create the AE in 3
    nvis, nhid = nhid_l1, nhid_l2
    path_ini_params_l2 = init_w_path + "dae_w_l2_init_" + str(nvis) + '_' +\
        str(nhid) + ".pkl"
    dae_l2 = DenoisingAutoencoder(dae_l1.encode(dae_l0.encode((input))),
                                  nvis=nhid_l1,
                                  nhid=nhid_l2,
                                  L1_reg=0.,
                                  L2_reg=0.,
                                  rnd=rnd,
                                  nonlinearity=NonLinearity.TANH,
                                  cost_type=CostType.MeanSquared,
                                  reverse=False)

    if not os.path.isfile(path_ini_params_l2):
        dae_l2.save_params(path_ini_params_l2)
    else:
        dae_l2.set_params_vals(path_ini_params_l2)

    # Create the AE out
    nvis, nhid = 68*2, nhid_l2
    path_ini_params_l3 = init_w_path + "dae_w_l3_init_" + str(nvis) + '_' +\
        str(nhid) + ".pkl"
    dae_l3 = DenoisingAutoencoder(output,
                                  L1_reg=0.,
                                  L2_reg=1e-2,
                                  nvis=nvis,
                                  nhid=nhid,
                                  rnd=rnd,
                                  nonlinearity=NonLinearity.TANH,
                                  cost_type=CostType.MeanSquared,
                                  reverse=True)

    if not os.path.isfile(path_ini_params_l3):
        dae_l3.save_params(path_ini_params_l3)
    else:
        dae_l3.set_params_vals(path_ini_params_l3)
    # Create the network
    rng = np.random.RandomState(23455)

    layer0 = {
        "rng": rng,
        "n_in": w*h,
        "n_out": nhid_l0,
        "W": dae_l0.hidden.W,
        "b": dae_l0.hidden.b,
        "activation": NonLinearity.SIGMOID
        }
    layer1 = {
        "rng": rng,
        "n_in": nhid_l0,
        "n_out": nhid_l1,
        "W": dae_l1.hidden.W,
        "b": dae_l1.hidden.b,
        "activation": NonLinearity.SIGMOID
        }
    layer2 = {
        "rng": rng,
        "n_in": nhid_l1,
        "n_out": nhid_l2,
        "W": dae_l2.hidden.W,
        "b": dae_l2.hidden.b,
        "activation": NonLinearity.TANH
        }
    layer3 = {
        "rng": rng,
        "n_in": nhid_l2,
        "n_out": 68*2,
        "W": dae_l3.hidden.W_prime,
        "b": dae_l3.hidden.b_prime,
        "activation": NonLinearity.TANH
        }
    layers = [layer0, layer1, layer2, layer3]
#    dropout = [float(sys.argv[1]), float(sys.argv[2]), float(sys.argv[3]),
#               float(sys.argv[4])]
    dropout = [0.0, 0.0, 0.0, 0.0]
    # number of the hidden layer just before the output ae. Default: None
    id_code = None
    model = ModelMLP(layers, input, l1_reg=0., l2_reg=0., reg_bias=False,
                     dropout=dropout, id_code=id_code)
    aes_in = [dae_l0, dae_l1]
    aes_out = []
    if id_code is not None:
        assert aes_out != []
    # Train
    # Data
    tr_batch_size = 10
    vl_batch_size = 8000

    with open(path_valid, 'r') as f:
        l_samples_vl = pkl.load(f)
    list_minibatchs_vl = split_data_to_minibatchs_eval(
        l_samples_vl, vl_batch_size)
    max_epochs = int(1000)
    lr_vl = 1e-3
    lr = sharedX_value(lr_vl, name="lr")
    # cost weights
    separate = True
    l_in = [sharedX_value(1., name="l_in"), sharedX_value(0.0, name="l_in2")]
    l_out = [sharedX_value(0., name="l_out")]
    l_sup = sharedX_value(1., name="l_sup")
    l_code = sharedX_value(0.0, name="l_code")
    if not separate:
        assert l_sup.get_value() + l_in.get_value() + l_out.get_value() == 1.
    if l_in[0].get_value() != 0. and aes_in == []:
        raise ValueError("You setup the l_in but no aes in found.")
    if l_out[0].get_value() != 0. and aes_out == []:
        raise ValueError("You setup the l_out but no aes out found.")
    # Train criterion

    cost_type = CostType.MeanSquared  # CostType.MeanSquared