from keras.models import Model, Sequential
from keras.layers import Input, LSTM, GRU, Dense, Embedding, Bidirectional, BatchNormalization
from keras import optimizers
from core.text_to_number import TextToNumber
from keras.utils import plot_model
from keras.callbacks import CSVLogger, ModelCheckpoint
import numpy as np

textToNumber = TextToNumber()

train_body_file = 'output/seq2seq/body'
train_title_file = 'output/seq2seq/title'

# load the processed files
encoder_input_data, doc_length = textToNumber.load_encoder_inputs(train_body_file + '_vecs.npy')
decoder_input_data, decoder_target_data = textToNumber.load_decoder_inputs(train_title_file + '_vecs.npy')

# load the preprocessor files
num_encoder_tokens, body_pp = textToNumber.load_text_processor(train_body_file + '_pp.dpkl')
num_decoder_tokens, title_pp = textToNumber.load_text_processor(train_title_file + '_pp.dpkl')



#arbitrarly set latent dimension for embedding and hidden units
latent_dim = 300

##### Define Model Architecture ######

########################
#### Encoder Model ####
encoder_inputs = Input(shape=(doc_length,), name='Encoder-Input')

# Word embeding for encoder (ex: Issue Body)
x = Embedding(num_encoder_tokens, latent_dim, name='Body-Word-Embedding', mask_zero=False)(encoder_inputs)
x = BatchNormalization(name='Encoder-Batchnorm-1')(x)

# Intermediate GRU layer (optional)
#x = GRU(latent_dim, name='Encoder-Intermediate-GRU', return_sequences=True)(x)
#x = BatchNormalization(name='Encoder-Batchnorm-2')(x)

# We do not need the `encoder_output` just the hidden state.
_, state_h = GRU(latent_dim, return_state=True, name='Encoder-Last-GRU')(x)

encoder_model = Model(inputs=encoder_inputs, outputs=state_h, name='Encoder-Model')

seq2seq_encoder_out = encoder_model(encoder_inputs)

########################
#### Decoder Model ####
decoder_inputs = Input(shape=(None,), name='Decoder-Input')  # for teacher forcing

# Word Embedding For Decoder (ex: Issue Titles)
dec_emb = Embedding(num_decoder_tokens, latent_dim, name='Decoder-Word-Embedding', mask_zero=False)(decoder_inputs)
dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb)

# Set up the decoder, using `decoder_state_input` as initial state.
decoder_gru = GRU(latent_dim, return_state=True, return_sequences=True, name='Decoder-GRU')
decoder_gru_output, _ = decoder_gru(dec_bn, initial_state=seq2seq_encoder_out)
x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output)

# Dense layer for prediction
decoder_dense = Dense(num_decoder_tokens, activation='softmax', name='Final-Output-Dense')
decoder_outputs = decoder_dense(x)

########################
#### Seq2Seq Model ####

#seq2seq_decoder_out = decoder_model([decoder_inputs, seq2seq_encoder_out])
seq2seq_Model = Model([encoder_inputs, decoder_inputs], decoder_outputs)


seq2seq_Model.compile(optimizer=optimizers.Nadam(lr=0.001), loss='sparse_categorical_crossentropy')

seq2seq_Model.summary()

plot_model(seq2seq_Model, to_file='output/seq2seq/model.png')


script_name_base = 'tutorial_seq2seq'
csv_logger = CSVLogger('output/seq2seq/{:}.log'.format(script_name_base))
model_checkpoint = ModelCheckpoint('output/seq2seq/{:}.epoch{{epoch:02d}}-val{{val_loss:.5f}}.hdf5'.format(script_name_base),
                                   save_best_only=True)

batch_size = 1200
epochs = 7
history = seq2seq_Model.fit([encoder_input_data, decoder_input_data], np.expand_dims(decoder_target_data, -1),
          batch_size=batch_size,
          epochs=epochs,
          validation_split=0.12, callbacks=[csv_logger, model_checkpoint])

#save model
seq2seq_Model.save('output/seq2seq/seq2seq_model_tutorial.h5')

print('done')

from keras.models import Model, Sequential
from keras.layers import Input, LSTM, GRU, Dense, Embedding, Bidirectional, BatchNormalization
from keras import optimizers
from core.text_to_number import TextToNumber
from keras.utils import plot_model
from keras.callbacks import CSVLogger, ModelCheckpoint
import numpy as np

textToNumber = TextToNumber()

train_body_file = 'output/seq2seq/body'
train_title_file = 'output/seq2seq/title'

# load the processed files
encoder_input_data, doc_length = textToNumber.load_encoder_inputs(train_body_file + '_vecs.npy')
decoder_input_data, decoder_target_data = textToNumber.load_decoder_inputs(train_title_file + '_vecs.npy')

# load the preprocessor files
num_encoder_tokens, body_pp = textToNumber.load_text_processor(train_body_file + '_pp.dpkl')
num_decoder_tokens, title_pp = textToNumber.load_text_processor(train_title_file + '_pp.dpkl')



#arbitrarly set latent dimension for embedding and hidden units
latent_dim = 300

##### Define Model Architecture ######

########################
#### Encoder Model ####
encoder_inputs = Input(shape=(doc_length,), name='Encoder-Input')

# Word embeding for encoder (ex: Issue Body)
x = Embedding(num_encoder_tokens, latent_dim, name='Body-Word-Embedding', mask_zero=False)(encoder_inputs)
x = BatchNormalization(name='Encoder-Batchnorm-1')(x)

# Intermediate GRU layer (optional)
#x = GRU(latent_dim, name='Encoder-Intermediate-GRU', return_sequences=True)(x)
#x = BatchNormalization(name='Encoder-Batchnorm-2')(x)

# We do not need the `encoder_output` just the hidden state.
_, state_h = GRU(latent_dim, return_state=True, name='Encoder-Last-GRU')(x)

encoder_model = Model(inputs=encoder_inputs, outputs=state_h, name='Encoder-Model')

seq2seq_encoder_out = encoder_model(encoder_inputs)

########################
#### Decoder Model ####
decoder_inputs = Input(shape=(None,), name='Decoder-Input')  # for teacher forcing

# Word Embedding For Decoder (ex: Issue Titles)
dec_emb = Embedding(num_decoder_tokens, latent_dim, name='Decoder-Word-Embedding', mask_zero=False)(decoder_inputs)
dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb)

# Set up the decoder, using `decoder_state_input` as initial state.
decoder_gru = GRU(latent_dim, return_state=True, return_sequences=True, name='Decoder-GRU')
decoder_gru_output, _ = decoder_gru(dec_bn, initial_state=seq2seq_encoder_out)
x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output)

# Dense layer for prediction
decoder_dense = Dense(num_decoder_tokens, activation='softmax', name='Final-Output-Dense')
decoder_outputs = decoder_dense(x)

########################
#### Seq2Seq Model ####

#seq2seq_decoder_out = decoder_model([decoder_inputs, seq2seq_encoder_out])
seq2seq_Model = Model([encoder_inputs, decoder_inputs], decoder_outputs)


seq2seq_Model.compile(optimizer=optimizers.Nadam(lr=0.001), loss='sparse_categorical_crossentropy')

seq2seq_Model.summary()

plot_model(seq2seq_Model, to_file='output/seq2seq/model.png')


script_name_base = 'tutorial_seq2seq'
csv_logger = CSVLogger('output/seq2seq/{:}.log'.format(script_name_base))
model_checkpoint = ModelCheckpoint('output/seq2seq/{:}.epoch{{epoch:02d}}-val{{val_loss:.5f}}.hdf5'.format(script_name_base),
                                   save_best_only=True)

batch_size = 1200
epochs = 7
history = seq2seq_Model.fit([encoder_input_data, decoder_input_data], np.expand_dims(decoder_target_data, -1),
          batch_size=batch_size,
          epochs=epochs,
          validation_split=0.12, callbacks=[csv_logger, model_checkpoint])

#save model
seq2seq_Model.save('output/seq2seq/seq2seq_model_tutorial.h5')

print('done')

# load the processed files # load the preprocessor files #arbitrarly set latent dimension for embedding and hidden units ##### Define Model Architecture ###### ######################## #### Encoder Model #### # Word embeding for encoder (ex: Issue Body) # Intermediate GRU layer (optional) #x = GRU(latent_dim, name='Encoder-Intermediate-GRU', return_sequences=True)(x) #x = BatchNormalization(name='Encoder-Batchnorm-2')(x) # We do not need the `encoder_output` just the hidden state. ######################## #### Decoder Model #### # for teacher forcing # Word Embedding For Decoder (ex: Issue Titles) # Set up the decoder, using `decoder_state_input` as initial state. # Dense layer for prediction ######################## #### Seq2Seq Model #### #seq2seq_decoder_out = decoder_model([decoder_inputs, seq2seq_encoder_out]) #save model