<gh_stars>1-10
import logging
import sys

sys.path.append('../../sr2018')
from components.data.vocabulary.morph_vocab_mlp import MorphMLPVocabulary
import copy
from components.utils.tensors import cuda_if_gpu
from torch.autograd import Variable
import torch
from components.data.base_morph_data import BaseMorphData
from components.utils.serialization import save_txt
from collections import Counter
from components.utils.readers import read_conll_data_file

logger = logging.getLogger('main')

class MorphMLPData(BaseMorphData):

    def set_vocabulary_class(self):
        self.vocab = MorphMLPVocabulary()

    def get_special_params_to_save(self):
        pd = {}
        pd['max_src_len'] = self.max_src_len
        pd['max_tgt_len'] = self.max_tgt_len
        pd['vocab_fn'] = self.vocab_fn
        return pd

    def prediction_setup(self):
        self.max_src_len = self.config['max_src_len']
        self.max_tgt_len = self.config['max_tgt_len']

        # else load vocabulary file from the filesystem
        assert 'vocab_fn' in self.config, logger.error('Specify "vocab_fn" in the config file')
        self.vocab.setup(vocab_path=self.vocab_fn,
                         data=None,
                         lower=self.lower,
                         source='file')

        self.set_num_features()

    def training_setup(self, train_data_fname, dev_data_fname):

        assert train_data_fname is not None
        assert dev_data_fname is not None

        train_raw = read_conll_data_file(train_data_fname)
        dev_raw = read_conll_data_file(dev_data_fname)

        # extract from TRAIN lemmas, forms, feats if in training mode
        train_lemmas_l, train_forms_l, train_feat_d = BaseMorphData.extract_lemmas_forms_feature_dicts(train_raw)
        dev_lemmas_l, dev_forms_l, dev_feat_d = BaseMorphData.extract_lemmas_forms_feature_dicts(dev_raw)
        self.set_char_freqlist(train_lemmas_l)

        self.max_src_len = self.config['max_src_len']
        self.max_tgt_len = self.config['max_tgt_len']

        logger.info('Max lemma/form lenghts: %d/%d', self.max_src_len, self.max_tgt_len)

        train_lemmas_l, train_forms_l, train_feat_d = self.filter_data_outliers(train_lemmas_l,
                                                                                train_forms_l,
                                                                                train_feat_d)

        dev_lemmas_l, dev_forms_l, dev_feat_d = self.filter_data_outliers(dev_lemmas_l,
                                                                          dev_forms_l,
                                                                          dev_feat_d)

        # vectorize the data
        self.vocab.setup(vocab_path=self.vocab_fn,
                         data=(train_lemmas_l, train_forms_l, train_feat_d),
                         lower=self.lower,
                         source='lemma_form_feat')

        self.set_num_features()

        # Vectorize TRAIN data
        self.train = self.train_data_to_ids(data=list(zip(
            train_lemmas_l,
            train_feat_d,
            train_forms_l)))

        self.dev = self.train_data_to_ids(data=list(zip(
            dev_lemmas_l,
            dev_feat_d,
            dev_forms_l)))

        # self.dev_references = dev_forms_l
        # save_txt(self.dev_references, self.fnames.dev_ref_fn)

    def set_num_features(self):
        self.num_features = len(self.vocab.src_vocab.featkey2idx)

    def train_data_to_ids(self, data):

        logger.debug('Vectorizing data')

        assert self.max_src_len is not None
        assert self.max_tgt_len is not None

        data_split_x = []
        data_split_y = []
        forms = []

        for idx, datum in enumerate(data):

            lemma, feat_dict, form = datum
            x_ids = MorphMLPData.feat_d_to_ids(feat_dict, self.vocab)
            lemma_id = self.vocab.src_vocab.lookup_tok(lemma)
            x_ids[0] = lemma_id
            y_id = self.vocab.tgt_vocab.lookup_tok(form)

            data_split_x.append(x_ids)
            data_split_y.append(y_id)
            forms.append(form)

        return (data_split_x, data_split_y, forms)

    def batchify_training_data(self, xy_ids, batch_size, is_dev_data):

        logger.debug('Batchifying data')

        x_data_ids, y_data_ids, forms = xy_ids
        data_size = len(x_data_ids)
        num_batches = data_size // batch_size
        data_indices = self.index_data(data_size, mode='no_shuffling')

        batch_pairs = []

        for bi in range(num_batches+1):
            batch_x = []
            batch_y = []

            curr_batch_indices = data_indices[bi * batch_size: (bi + 1) * batch_size]

            for idx in curr_batch_indices:
                x_ids = x_data_ids[idx]
                y_ids = y_data_ids[idx]

                x_enc_ids_copy = copy.deepcopy(x_ids)
                batch_x.append(x_enc_ids_copy)

                y_ids_copy = copy.deepcopy(y_ids)
                batch_y.append(y_ids_copy)

            batch_enc_x_var = cuda_if_gpu(Variable(torch.LongTensor(batch_x)))
            batch_dec_y_var = cuda_if_gpu(Variable(torch.LongTensor(batch_y)))

            batch_pairs.append((batch_enc_x_var, batch_dec_y_var))

        if is_dev_data:
            self.dev_references = forms
            logger.info('Saving dev (training) references to --> %s', self.fnames.dev_ref_fn)
            save_txt(self.dev_references, self.fnames.dev_ref_fn)

        assert data_size == len(forms)

        return batch_pairs

    @staticmethod
    def feat_d_to_ids(feat_d, vocab):
        feat_vecs = vocab.src_vocab.dummy_feat_vec
        for feat_type, idx in vocab.src_vocab.featkey2idx.items():
            if feat_type in feat_d:
                feature = vocab.src_vocab.feat_from_kvpair(feat_type, feat_d[feat_type])
                feat_vecs[idx] = vocab.src_vocab.lookup_tok(feature)

        return feat_vecs

    @staticmethod
    def unvectorize_seq(id_seq, vocab):
        return vocab.tgt_vocab.unvectorize_id_seq(id_seq)



component = MorphMLPData

import logging
import sys

sys.path.append('../../sr2018')
from components.data.vocabulary.morph_vocab_mlp import MorphMLPVocabulary
import copy
from components.utils.tensors import cuda_if_gpu
from torch.autograd import Variable
import torch
from components.data.base_morph_data import BaseMorphData
from components.utils.serialization import save_txt
from collections import Counter
from components.utils.readers import read_conll_data_file

logger = logging.getLogger('main')

class MorphMLPData(BaseMorphData):

    def set_vocabulary_class(self):
        self.vocab = MorphMLPVocabulary()

    def get_special_params_to_save(self):
        pd = {}
        pd['max_src_len'] = self.max_src_len
        pd['max_tgt_len'] = self.max_tgt_len
        pd['vocab_fn'] = self.vocab_fn
        return pd

    def prediction_setup(self):
        self.max_src_len = self.config['max_src_len']
        self.max_tgt_len = self.config['max_tgt_len']

        # else load vocabulary file from the filesystem
        assert 'vocab_fn' in self.config, logger.error('Specify "vocab_fn" in the config file')
        self.vocab.setup(vocab_path=self.vocab_fn,
                         data=None,
                         lower=self.lower,
                         source='file')

        self.set_num_features()

    def training_setup(self, train_data_fname, dev_data_fname):

        assert train_data_fname is not None
        assert dev_data_fname is not None

        train_raw = read_conll_data_file(train_data_fname)
        dev_raw = read_conll_data_file(dev_data_fname)

        # extract from TRAIN lemmas, forms, feats if in training mode
        train_lemmas_l, train_forms_l, train_feat_d = BaseMorphData.extract_lemmas_forms_feature_dicts(train_raw)
        dev_lemmas_l, dev_forms_l, dev_feat_d = BaseMorphData.extract_lemmas_forms_feature_dicts(dev_raw)
        self.set_char_freqlist(train_lemmas_l)

        self.max_src_len = self.config['max_src_len']
        self.max_tgt_len = self.config['max_tgt_len']

        logger.info('Max lemma/form lenghts: %d/%d', self.max_src_len, self.max_tgt_len)

        train_lemmas_l, train_forms_l, train_feat_d = self.filter_data_outliers(train_lemmas_l,
                                                                                train_forms_l,
                                                                                train_feat_d)

        dev_lemmas_l, dev_forms_l, dev_feat_d = self.filter_data_outliers(dev_lemmas_l,
                                                                          dev_forms_l,
                                                                          dev_feat_d)

        # vectorize the data
        self.vocab.setup(vocab_path=self.vocab_fn,
                         data=(train_lemmas_l, train_forms_l, train_feat_d),
                         lower=self.lower,
                         source='lemma_form_feat')

        self.set_num_features()

        # Vectorize TRAIN data
        self.train = self.train_data_to_ids(data=list(zip(
            train_lemmas_l,
            train_feat_d,
            train_forms_l)))

        self.dev = self.train_data_to_ids(data=list(zip(
            dev_lemmas_l,
            dev_feat_d,
            dev_forms_l)))

        # self.dev_references = dev_forms_l
        # save_txt(self.dev_references, self.fnames.dev_ref_fn)

    def set_num_features(self):
        self.num_features = len(self.vocab.src_vocab.featkey2idx)

    def train_data_to_ids(self, data):

        logger.debug('Vectorizing data')

        assert self.max_src_len is not None
        assert self.max_tgt_len is not None

        data_split_x = []
        data_split_y = []
        forms = []

        for idx, datum in enumerate(data):

            lemma, feat_dict, form = datum
            x_ids = MorphMLPData.feat_d_to_ids(feat_dict, self.vocab)
            lemma_id = self.vocab.src_vocab.lookup_tok(lemma)
            x_ids[0] = lemma_id
            y_id = self.vocab.tgt_vocab.lookup_tok(form)

            data_split_x.append(x_ids)
            data_split_y.append(y_id)
            forms.append(form)

        return (data_split_x, data_split_y, forms)

    def batchify_training_data(self, xy_ids, batch_size, is_dev_data):

        logger.debug('Batchifying data')

        x_data_ids, y_data_ids, forms = xy_ids
        data_size = len(x_data_ids)
        num_batches = data_size // batch_size
        data_indices = self.index_data(data_size, mode='no_shuffling')

        batch_pairs = []

        for bi in range(num_batches+1):
            batch_x = []
            batch_y = []

            curr_batch_indices = data_indices[bi * batch_size: (bi + 1) * batch_size]

            for idx in curr_batch_indices:
                x_ids = x_data_ids[idx]
                y_ids = y_data_ids[idx]

                x_enc_ids_copy = copy.deepcopy(x_ids)
                batch_x.append(x_enc_ids_copy)

                y_ids_copy = copy.deepcopy(y_ids)
                batch_y.append(y_ids_copy)

            batch_enc_x_var = cuda_if_gpu(Variable(torch.LongTensor(batch_x)))
            batch_dec_y_var = cuda_if_gpu(Variable(torch.LongTensor(batch_y)))

            batch_pairs.append((batch_enc_x_var, batch_dec_y_var))

        if is_dev_data:
            self.dev_references = forms
            logger.info('Saving dev (training) references to --> %s', self.fnames.dev_ref_fn)
            save_txt(self.dev_references, self.fnames.dev_ref_fn)

        assert data_size == len(forms)

        return batch_pairs

    @staticmethod
    def feat_d_to_ids(feat_d, vocab):
        feat_vecs = vocab.src_vocab.dummy_feat_vec
        for feat_type, idx in vocab.src_vocab.featkey2idx.items():
            if feat_type in feat_d:
                feature = vocab.src_vocab.feat_from_kvpair(feat_type, feat_d[feat_type])
                feat_vecs[idx] = vocab.src_vocab.lookup_tok(feature)

        return feat_vecs

    @staticmethod
    def unvectorize_seq(id_seq, vocab):
        return vocab.tgt_vocab.unvectorize_id_seq(id_seq)



component = MorphMLPData
# else load vocabulary file from the filesystem # extract from TRAIN lemmas, forms, feats if in training mode # vectorize the data # Vectorize TRAIN data # self.dev_references = dev_forms_l # save_txt(self.dev_references, self.fnames.dev_ref_fn)