def get_minibatches_idx(n, minibatch_size, shuffle=False):
    """
    Used to shuffle the dataset at each iteration.
    """

    idx_list = numpy.arange(n, dtype="int32")

    if shuffle:
        numpy.random.shuffle(idx_list)

    minibatches = []
    minibatch_start = 0
    for i in range(n // minibatch_size):
        minibatches.append(idx_list[minibatch_start:
                                    minibatch_start + minibatch_size])
        minibatch_start += minibatch_size

    if (minibatch_start != n):
        # Make a minibatch out of what is left
        minibatches.append(idx_list[minibatch_start:])

    return zip(range(len(minibatches)), minibatches) def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):

    if allow_append:
        build_character_dictionary(path, chardict)
        build_word_dictionary(path, worddict)
        build_tag_dictionary(path, posdict)

    cur_chars, cur_words, cur_labels = [], [], []
    words, chars, labels = [], [], []
    with open(path, 'r') as fin:
        for line in fin:
            line = line.strip()
            if len(line) == 0:
                chars.append(cur_chars[:-1])
                words.append(cur_words[:-1])
                labels.append(cur_labels)
                cur_chars = []
                cur_labels = []
                cur_words = []
                continue

            word, pos = line.split('\t')

            if word not in worddict and allow_append:
                worddict[word] = len(worddict)+1

            for c in word:
                if c not in chardict and allow_append:
                    chardict[c] = len(chardict)+1

                if c in chardict:
                    cur_chars.append(chardict[c])
                else:
                    cur_chars.append(0)

                if word in worddict:
                    cur_words.append(worddict[word])
                else:
                    cur_words.append(0)

                if pos not in posdict and allow_append:
                    posdict[pos] = len(posdict)+1

            if pos in posdict:
                cur_labels.append(posdict[pos])
            else:
                cur_labels.append(0)

            if word in worddict:
                cur_words.append(worddict[word])
            else:
                cur_words.append(0)
            cur_chars.append(0)

    if len(cur_chars) > 0:
    	chars.append(cur_chars)
        words.append(cur_words)
    	labels.append(cur_labels)

    return chars, words, labels def prepare_data(char_seqs, labels, maxw, maxwlen, dim_proj):
    """
    Create the matrices from the datasets.

    This pad each sequence to the same length: the length of the
    longest sequence or maxlen.

    if maxlen is set, we will cut all sequences to this maximum
    length

    This swap the axis!
    """

    # x: a list of sentences
    n_samples = len(char_seqs)

    x_c = numpy.zeros((maxw, maxwlen, n_samples)).astype('int8')
    x_mask = numpy.zeros((maxw, maxwlen, n_samples, dim_proj)).astype(theano.config.floatX)
    y = numpy.zeros((maxw, n_samples)).astype('int8')
    y_mask = numpy.zeros((maxw, n_samples)).astype('int8')

    for idx, (s_c, l) in enumerate(zip(char_seqs, labels)):
        # idx is the current position in the mini-batch
        # s_c is a list of characters
        # s_w is a list of words
        # l is a list of labels
        c = 0
        p = 0
        warning = None
        for j, a in enumerate(s_c):
            # j is the current character in this tweet
            # idx is the current tweet in this minibatch
            # c is the current word (can be up to 16)
            # p is the current character in this word

            if a == 0:
                # This current character is a space
                # Increase the word count and continue
                c += 1
                p = 0
                j += 1 # Temporarily skip to next loop char
                if c >= maxw:
                    if j != len(s_c):
                        warning = "truncation: too many words in this tweet! {}-{}".format(j, len(s_c))
                    break
                if c >= len(l):
                    if j != len(s_c):
                        warning = "truncation: too many words for these labels {}-{}".format(j, len(s_c))
                    break

            if p >= x_c.shape[1]:
                warning = "truncation: too many characters for this maxwlen"
            else:
                x_c[c, p, idx] = a
                x_mask[c, p, idx] = numpy.ones(dim_proj)

            y[c, idx] = l[c]
            y_mask[c, idx] = 1
            p += 1

        if warning is not None:
            #logging.warning("%s", warning)
            pass

    return x_c, x_mask, y, y_mask def get_max_word_count(path):
    t = get_tweet_words(path)
    m = [len(t[c]) for c in t]
    m = int(numpy.percentile(m, 99))
    #m = int(numpy.median([len(t[c]) for c in t]))
    logging.debug("get_max_word_count('%s') = %d", path, m)
    return m def get_max_length(path):
    t = get_tweet_words(path)
    t = {c: u"".join(t[c]) for c in t}
    m = max([len(t[c]) for c in t])
    logging.debug('get_max_length(%s) = %d', path, m)
    return m def get_max_word_length(path):
    t = get_tweet_words(path)
    m = 0
    d = []
    for c in t:
        for w in t[c]:
            d.append(len(w))
            if len(w) >= m:
                m = len(w)
                logging.debug('length: %s, %d', w, m)
    m = numpy.percentile(d, 99)
    logging.debug("get_max_word_length('%s') = %d", path, m)
    return m def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None, go_backwards=False, mult=1):
    """

    :param tparams:
    :param state_below:
    :param options:
    :param prefix:
    :param mask:
    :return:
    """
    nsteps = state_below.shape[0]
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
    else:
        n_samples = 1

    assert mask is not None

    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n * dim:(n + 1) * dim]
        return _x[:, n * dim:(n + 1) * dim]

    def _p(pp, name):
        return '%s_%s' % (pp, name)

    def _step(m_, x_, h_, c_):

        preact = tensor.dot(h_, tparams[_p(prefix, 'U')])
        preact += x_

        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']))

        c = f * c_ + i * c
        c = m_[:, None] * c + (1. - m_)[:, None] * c_

        h = o * tensor.tanh(c)
        h = m_[:, None] * h + (1. - m_)[:, None] * h_

        return h, c

    state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) +
                   tparams[_p(prefix, 'b')])

    dim_proj = options['dim_proj'] * mult
    rval, updates = theano.scan(_step,
                                sequences=[mask, state_below],
                                outputs_info=[tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj),
                                              tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj)],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps, go_backwards=go_backwards)
    return rval[0] def lstm_unmasked_layer(tparams, state_below, options, prefix='lstm', mult=1, go_backwards=False):
    """

    :param tparams:
    :param state_below:
    :param options:
    :param prefix:
    :return:
    """
    nsteps = state_below.shape[0]
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
    else:
        n_samples = 1

    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n * dim:(n + 1) * dim]
        return _x[:, n * dim:(n + 1) * dim]

    def _p(pp, name):
        return '%s_%s' % (pp, name)

    def _step(x_, h_, c_):

        preact = tensor.dot(h_, tparams[_p(prefix, 'U')])
        preact += x_

        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']*mult))
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']*mult))
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']*mult))
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']*mult))

        c = f * c_ + i * c

        h = o * tensor.tanh(c)

        return h, c

    state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) +
                   tparams[_p(prefix, 'b')])

    dim_proj = options['dim_proj']*mult
    rval, updates = theano.scan(_step,
                                sequences=[state_below],
                                outputs_info=[tensor.cast(tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj), theano.config.floatX),
                                              tensor.cast(tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj), theano.config.floatX)],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps,
                                go_backwards=go_backwards)
    return rval[0] def bidirectional_lstm_layer(tparams, state_below, options, prefix='lstm', mask=None, mult=1):

    def _p(pp, name):
        return '%s_%s' % (pp, name)

    prefix_forwards = '%s_forwards' % (prefix,)
    prefix_backwards = '%s_backwards' % (prefix,)

    if mask is not None:
        forwards = lstm_layer(tparams, state_below, options, prefix=prefix_forwards, mask=mask, go_backwards=False, mult=mult)
        backwards = lstm_layer(tparams, state_below, options, prefix=prefix_backwards, mask=mask, go_backwards=True, mult=mult)
    else:
        forwards = lstm_unmasked_layer(tparams, state_below, options, prefix=prefix_forwards, mult=mult, go_backwards=False)
        backwards = lstm_unmasked_layer(tparams, state_below, options, prefix=prefix_backwards, mult=mult, go_backwards=True)

    #forwards = theano.printing.Print(prefix_forwards, attrs=["shape"])(forwards)
    #backwards = theano.printing.Print(prefix_forwards, attrs=["shape"])(backwards)

    return forwards + backwards def pred_error(f_pred, prepare_data, data, iterator, maxw, max_word_length, dim_proj):
    """
    Just compute the error
    f_pred: Theano fct computing the prediction
    prepare_data: usual prepare_data for that dataset.
    """
    valid_err = []
    valid_shapes = []
    for _, valid_index in iterator:
        xc, mask, y, y_mask = prepare_data([data[0][t] for t in valid_index],
                                                 numpy.array(data[2])[valid_index],
                                                 maxw, max_word_length, dim_proj)
        preds = f_pred(xc, mask, y_mask)
        preds = preds[numpy.nonzero(y)]
        acc = numpy.equal(preds, y[numpy.nonzero(y)])
        valid_shapes.append(preds.size)
        valid_err.append(acc.sum())

    valid_err = 1. - 1.0*numpy.asarray(valid_err).sum() / numpy.asarray(valid_shapes).sum()

    return valid_err def zipp(params, tparams):
    """
    When we reload the model. Needed for the GPU stuff.
    """
    for kk, vv in params.iteritems():
        tparams[kk].set_value(vv) def unzip(zipped):
    """
    When we pickle the model. Needed for the GPU stuff.
    """
    new_params = OrderedDict()
    for kk, vv in zipped.iteritems():
        new_params[kk] = vv.get_value()
    return new_params def load_params(path, params):
    logging.info("Loading model from file '%s'...", path)
    pp = numpy.load(path)
    for k in pp:
        params[k] = pp[k]
    path = "%s.pkl" % (path,)
    logging.info("Loading model from file '%s'...", path)
    with open(path, 'rb') as fin:
        data = pickle.load(fin)
        for k in ['dim_proj_chars', 'char_dict', 'pos_dict', 'word_dict']:
            params[k] = data[k]
    return params
from argparse import ArgumentParser
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from theano.tensor.shared_randomstreams import RandomStreams
from util import get_minibatches_idx
from modelio import load_pos_tagged_data, prepare_data, get_max_word_count, get_max_length, get_max_word_length
from nn_layers import *
from nn_lstm import lstm_layer, lstm_unmasked_layer, bidirectional_lstm_layer
from nn_params import *
from nn_optimizers import *
from nn_support import pred_error
from nn_serialization import zipp, unzip, load_params
import logging
import itertools
import cPickle as pkl
import time
import os.path
import pickle
    for i, ((c, w), l) in enumerate(zip(zip(src[0], src[1]), src[2])):
        if i < fin:
            val_chars.append(c)
            val_words.append(w)
            val_labels.append(l)
        else:
            src_chars.append(c)
            src_words.append(w)
            src_labels.append(l)
    return (src_chars, src_words, src_labels), (val_chars, val_words, val_labels)

def train_lstm(
    dim_proj_chars=32,  # character embedding dimension and LSTM number of hidden units.
    patience=4,  # Number of epoch to wait before early stop if no progress
    max_epochs=5000,  # The maximum number of epoch to run
    dispFreq=10,  # Display to stdout the training progress every N updates
    decay_c=0.0001,  # Weight decay for the classifier applied to the U weights.
    lrate=0.0001,  # Learning rate for sgd (not used for adadelta and rmsprop)
    optimizer=adadelta,  # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).
    encoder='lstm',  # TODO: can be removed must be lstm.
    saveto='lstm_model.npz',  # The best model will be saved there
    validFreq=900,  # Compute the validation error after this number of update.
    saveFreq=2220,  # Save the parameters after every saveFreq updates
    maxlen=100,  # Sequence longer then this get ignored
    batch_size=50,  # The batch size during training.
    valid_batch_size=64,  # The batch size used for validation/test set.
    dataset='imdb',

    # Parameter for extra option
    noise_std=0.,
    use_dropout=True,  # if False slightly faster, but worst test error
                       # This frequently need a bigger model.
    reload_model=None,  # Path to a saved model we want to start from.
    test_size=-1,  # If >0, we keep only this number of test example.
    pretrain = False, # If True, load some data from this argument
    # Use to keep track of feature enumeration
    char_dict = {},
    word_dict = {},
    pos_dict = {},
    word_layers=0,
    letter_layers=0
):

    # Model options
    model_options = locals().copy()
    print "model options", model_options

    if reload_model is not None:
        load_params(reload_model, model_options)
        char_dict = model_options['char_dict']
        word_dict = model_options['word_dict']
        pos_dict = model_options['pos_dict']

    # Load the training data
    print 'Loading data'

    input_path = "Data/Gate-Train.conll"

    load_pos_tagged_data(input_path, char_dict, word_dict, pos_dict)

    with open("substitutions.pkl", "rb") as fin:
        word_dict = pickle.load(fin)

    max_word_count = 0
    max_word_length = 0
    max_length = 0
    # Now load the data for real
    data = load_pos_tagged_data(input_path, char_dict, word_dict, pos_dict, 0)
    train, eval = split_at(data, 0.05)
    test, valid = split_at(eval, 0.50)
    max_word_count = max(max_word_count, \
    get_max_word_count(input_path))
    max_word_length = max(max_word_length, \
    get_max_word_length(input_path))
    max_length = max(max_word_length, get_max_length(input_path))

    #ydim = numpy.max(numpy.max(train[2])) + 1
    #print numpy.max(train[2])
    ydim = max(itertools.chain.from_iterable(train[2])) + 1
    print "ydim =", ydim

    model_options['ydim'] = ydim
    model_options['n_chars'] = len(char_dict)+1
    model_options['n_words'] = len(word_dict)+1

    model_options['word_dict'] = word_dict
    model_options['char_dict'] = char_dict
    model_options['pos_dict'] = pos_dict

    # This create the initial parameters as numpy ndarrays.
    # Dict name (string) -> numpy ndarray
    params = init_params(model_options, reload_model is not None)

    logging.info('Building model')

    # This create Theano Shared Variable from the parameters.
    # Dict name (string) -> Theano Tensor Shared Variable
    # params and tparams have different copy of the weights.
    tparams = init_tparams(params)

    # use_noise is for dropout
    (xc, mask,
     y, y_mask, f_pred_prob, f_pred, cost) = build_model(tparams, model_options, max_word_count)

    if decay_c > 0.:
        decay_c = theano.shared(numpy_floatX(decay_c), name='decay_c')
        weight_decay = 0.
        weight_decay += (tparams['U'] ** 2).sum()
        weight_decay *= decay_c
        cost += weight_decay

    f_cost = theano.function([xc, mask, y, y_mask], cost, name='f_cost', on_unused_input='warn')

    grads = tensor.grad(cost, wrt=tparams.values())
    f_grad = theano.function([xc, mask, y, y_mask], grads, name='f_grad', on_unused_input='warn')

    lr = tensor.scalar(name='lr')
    f_grad_shared, f_update = optimizer(lr, tparams, grads,
                                        xc, mask, y, y_mask, cost)


    kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size)