# -*- coding: utf-8 -*-
"""Copy of FC_blocks

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1ulQthlWW_8ZFMprtaZeroWRlO1YIekpl
"""

import sys
import numpy as np
import re
from numpy.random import rand as np_rand
from numpy.random import seed as np_seed
from numpy import array as np_array
from nltk.tokenize.casual import TweetTokenizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn import metrics

from keras.models import Sequential, Model, load_model
from keras.layers import LSTM, GRU, CuDNNLSTM, CuDNNGRU
from keras.layers import BatchNormalization
from keras import optimizers
from keras.layers.core import Dense, Activation, Flatten, Reshape
from keras.layers import Add, Multiply
from keras.layers.core import Dropout
from keras.layers import Input, RepeatVector
from keras.layers.core import Flatten
from keras.layers import Bidirectional, TimeDistributed
from keras.preprocessing import sequence

from keras.layers.embeddings import Embedding
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

from tqdm import tqdm



"""#Funciones"""

def read_embeddings(path, offset, random_state=42):
    """Load embeddings file.
    """
    word_embeddings = [[] for i in range(offset)]
    word_indexes = {}
    with open(path, "r", encoding="utf-8") as emb_file:
        emb_file.readline()
        for line in emb_file:
            fields = line.partition(EMB_SEP_CHAR)
            word = fields[0].strip()
            own_strip = str.strip
            emb_values = np_array([float(x) for x in own_strip(fields[-1]).split(EMB_SEP_CHAR)])
            word_indexes[word] = len(word_embeddings)
            word_embeddings.append(emb_values)
            
    # Offset = 2; Padding and OOV.
    np_seed(random_state)
    word_embeddings[0] = 2 * 0.1 * np_rand(len(word_embeddings[2])) - 1
    word_embeddings[1] = 2 * 0.1 * np_rand(len(word_embeddings[2])) - 1

    return (word_embeddings, word_indexes)

def tokenize(text):
    """Tokenize an input text
    
    Args:
        text: A String with the text to tokenize
    
    Returns:
        A list of Strings (tokens)
    """
    text_tokenized = TWEET_TOKENIZER.tokenize(text)
    return text_tokenized

def fit_transform_vocabulary_pretrain_embeddings(corpus, pre_embeddings_index):
    """Creates the vocabulary of the corpus.
        Index 0: padding
        Index 1: OOV.
    
    Args:
        corpus: A list os str (documents)
        
    Returns:
        A tuple whose first element is a dictionary word-index and the second
        element is a list of list in which each position is the index of the 
        token in the vocabulary
    """
    
    vocabulary = {}
    corpus_indexes = []
    corpus_indexes_append = corpus_indexes.append
    index = 0
    own_lowercase = str.lower
    for doc in tqdm(corpus, "Fitting tokenizer"):
        doc_indexes = []
        tokens = tokenize(own_lowercase(doc))
        for token in tokens:
            if RE_TOKEN_USER.fullmatch(token):
                token = "@user"
            if token in pre_embeddings_index:
                index = pre_embeddings_index[token]
                doc_indexes.append(index)
                if token not in vocabulary:
                    vocabulary[token] = index
            else:
                index = 1
                doc_indexes.append(index)
                if token not in vocabulary:
                    vocabulary[token] = index
        corpus_indexes_append(doc_indexes)
        
    return (vocabulary, corpus_indexes)

def calculate_quality_performance(y_labels, y_classified_labels, model_name):
    classes_index = [CLASSES.index(c) for c in CLASSES]
    accruacy = metrics.accuracy_score(y_labels, y_classified_labels)
    macro_precision = metrics.precision_score(y_labels, y_classified_labels, labels=classes_index, average="macro")
    macro_recall = metrics.recall_score(y_labels, y_classified_labels, labels=classes_index, average="macro")
    macro_f1 = metrics.f1_score(y_labels, y_classified_labels, labels=classes_index, average="macro")
    
    print("\n*** Results " + model_name + "***")
    print("Macro-Precision: " + str(macro_precision))
    print("Macro-Recall: " + str(macro_recall))
    print("Macro-F1: " + str(macro_f1))
    print("Accuracy: " + str(accruacy))

from google.colab import drive
drive.mount('/content/gdrive')

TWEET_TOKENIZER = TweetTokenizer(preserve_case=False, reduce_len=True, strip_handles=False)
CLASSES = []
EMB_SEP_CHAR = ' '
RE_TOKEN_USER = re.compile(r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){20}(?!@))|(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){1,19})(?![A-Za-z0-9_]*@)")


CLASSES = {'N':0, 'P':1, 'NEU':2, 'NONE':3}
CLASSES_inv = {v:k for k,v in CLASSES.items()}

# word2vec_path = '/content/fasttext_spanish_twitter_100d.vec'
word2vec_path = '/content/gdrive/My Drive/Colab Notebooks/fasttext_spanish_twitter_100d.vec'

import requests
import xml.etree.ElementTree as ET
import pandas as pd

# Primero cargamos los datos (ya los he subido a GH)
train_tagged_url = 'https://raw.githubusercontent.com/FGonzalezLopez/DM_AA_SentAnalysisTwitter/master/intertass-ES-train-tagged.xml'
validation_tagged_url = 'https://raw.githubusercontent.com/FGonzalezLopez/DM_AA_SentAnalysisTwitter/master/intertass-ES-development-tagged.xml'
test_url = 'https://raw.githubusercontent.com/FGonzalezLopez/DM_AA_SentAnalysisTwitter/master/intertass-ES-test.xml'

train_xml = ET.XML(requests.get(train_tagged_url).content)
val_xml = ET.XML(requests.get(validation_tagged_url).content)
test_xml = ET.XML(requests.get(test_url).content)

def parse_tweet(in_entry):
  # No sé si habrá una forma mejor (supongo que sí)
  vals_dict = dict()
  for attr in in_entry:
    if(not attr.tag == 'sentiment'):
      vals_dict[attr.tag] = attr.text
    else:
      vals_dict[attr.tag] = attr[0][0].text
          
  return vals_dict
  
train_df = pd.DataFrame.from_dict([parse_tweet(tweet) for tweet in tqdm(train_xml, "Parsing train tweets")])
val_df = pd.DataFrame.from_dict([parse_tweet(tweet) for tweet in tqdm(val_xml, "Parsing validation tweets")])
test_df = pd.DataFrame.from_dict([parse_tweet(tweet) for tweet in tqdm(test_xml, "Parsing test tweets")])

# Hacemos un shuffle (Importante!)
train_df = train_df.sample(frac=1.0).reset_index(drop=True)
val_df = val_df.sample(frac=1.0).reset_index(drop=True)

train_labels = train_df['sentiment'].values
tweets_train = train_df['content'].values
train_labels = np.array([CLASSES[c] for c in train_labels])

validation_labels = val_df['sentiment'].values
validation_labels = np.array([CLASSES[c] for c in validation_labels])
tweets_val = val_df['content'].values

tweets_test = test_df['content'].values

train_counts = np.unique(train_labels, return_counts=True)[1]
val_counts = np.unique(validation_labels, return_counts=True)[1]
print(train_counts/np.sum(train_counts))
print(val_counts/np.sum(val_counts))

def get_tokens(input_texts, word_index):
  corpus_tokens = []
  oov_tokens = 0
  for t in tqdm(input_texts, "Tokenize tweets"):
    tokens = tokenize(str.lower(t))
    t_indices = []
    for token in tokens:
      if(RE_TOKEN_USER.fullmatch(token) is not None):
        token = "@user"
      idx = word_index.get(token,1)
      if(idx == 1):
        oov_tokens+=1
      t_indices.append(idx)
    corpus_tokens.append(t_indices)
  return corpus_tokens, oov_tokens

def get_features(tweetsTrain, tweetsValidation, tweetsTest, embeddings_path, random_state=None):
    """Classification with RNN and embedings (pre-trained)
    """
    
    #Offset = 2; Padding and OOV.
    print("Begin loading embeddings.")
    word_embeddings, word_emb_indices = read_embeddings(embeddings_path, 2)
    print("End loading embeddings.")
    np_seed(random_state)
    
    
    #Build vocabulary and corpus indexes
    print("Computing training tokens")
    vocabulary_train, corpus_train_index = fit_transform_vocabulary_pretrain_embeddings(tweetsTrain, word_emb_indices)
    
    # Set a max input length
    max_len_input = int(np.percentile([len(tweet_train) for tweet_train in corpus_train_index], 95, axis=0))
    
    # Get the tokens for the validation and test sets
    print("Computing validation tokens")
    corpus_validation_index, oov_validation = get_tokens(tweetsValidation, word_emb_indices)
    print("OOV validation: %d" % oov_validation)
#     print(np.unique([el for sub in corpus_validation_index for el in sub], return_counts=True))
    
    print("Computing test tokens")
    corpus_test_index, oov_test = get_tokens(tweetsTest, word_emb_indices)
    print("OOV test: %d" % oov_test)
#     print(np.unique([el for sub in corpus_test_index for el in sub], return_counts=True))
    
    # Pad the train, validation and test token sequences
    train_features_pad = sequence.pad_sequences(corpus_train_index, maxlen=max_len_input, padding="post", truncating="post", dtype=type(corpus_train_index[0][0]))
    validation_features_pad = sequence.pad_sequences(corpus_validation_index, maxlen=max_len_input, padding="post", truncating="post", dtype=type(corpus_validation_index[0][0]))
    test_features_pad = sequence.pad_sequences(corpus_test_index, maxlen=max_len_input, padding="post", truncating="post", dtype=type(corpus_test_index[0][0]))
    
    return train_features_pad, validation_features_pad, test_features_pad, word_embeddings, word_emb_indices

def get_embedding_features(word_embedding_weights, in_data):
  # Separamos la extracción de características del embedding...
  embedding_model = Sequential()
  embedding_model.add((Embedding(len(word_embedding_weights), len(word_embedding_weights[0]), weights=[np_array(word_embedding_weights)], input_length=in_data.shape[1], trainable=False)))

  return embedding_model.predict(in_data, batch_size=256, verbose=1)

# Sacamos las features
train_feats, validation_feats, test_feats, embedding_weights, embedding_indices = get_features(tweets_train, tweets_val, tweets_test, word2vec_path, random_state=42)

train_feats_embedding = get_embedding_features(embedding_weights, train_feats)
validation_feats_embedding = get_embedding_features(embedding_weights, validation_feats)
test_feats_embedding = get_embedding_features(embedding_weights, test_feats)

def fc_module(in_tensor, units, drop_rate = 0.2, activation = 'elu', regularizer = 'l2'):
  out_tensor = in_tensor
  out_tensor = Dense(units, activation='linear', use_bias=False)(out_tensor)
  out_tensor = BatchNormalization(axis=-1)(out_tensor)
  out_tensor = Activation(activation)(out_tensor)
  out_tensor = Dropout(drop_rate)(out_tensor)
  return out_tensor

def get_model(in_data_shape):
  
  x_input = Input(shape = in_data_shape)
  
  x_output = x_input
  x_output = Dropout(0.4)(x_output)
  
#   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
  
#   # ATENSIÓ
#   attention_mask = fc_module(x_output, 10)
#   attention_mask = fc_module(x_output, 5)
#   attention_mask = Dense(1, kernel_regularizer='l2')(attention_mask)
#   attention_mask = Flatten()(attention_mask)
#   attention_mask = Activation('softmax')(attention_mask)
#   attention_mask = Reshape((-1,1))(attention_mask)
#   context = Multiply()([x_output, attention_mask])
  
#   x_output = Add()([x_output, context])
  
#   x_res = x_output
#   x_output = Dropout(0.2)(x_output)
#   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
#   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
#   x_output = Dropout(0.2)(x_output)
#   x_output = Add()([x_output, x_res])
  
  x_output = Bidirectional(CuDNNLSTM(10, return_sequences=False, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
  
  x_res = x_output
  x_output = Dropout(0.2)(x_output)
  x_output = fc_module(x_output, 20)
  x_output = Add()([x_output, x_res])
  x_output = Dropout(0.2)(x_output)
  x_output = fc_module(x_output, 20)
  x_res = x_output
  x_output = fc_module(x_output, 20)
  x_output = Add()([x_output, x_res])
  x_output = Dropout(0.2)(x_output)


  x_output = Dense(len(CLASSES), activation='softmax', kernel_regularizer='l2')(x_output)
  
  model = Model(inputs=x_input, outputs = x_output)
  
  return model

get_model((28,100)).summary()

def train_model(feats_train, labels_train, feats_validation, labels_validation, feats_test, word_embedding_weights, save_prefix = "", random_state=None, normalize_feats=True,):
    """Classification with RNN and embedings (pre-trained)
    """

    if(normalize_feats):
      # Normalizamos las features
      feats_train = feats_train - np.mean(feats_train, axis=2, keepdims=True)
      feats_validation = feats_validation - np.mean(feats_validation, axis=2, keepdims=True)
      feats_test = feats_test - np.mean(feats_test, axis=2, keepdims=True)

      feats_train = feats_train / np.std(feats_train, axis=2, keepdims=True)
      feats_validation = feats_validation / np.std(feats_validation, axis=2, keepdims=True)
      feats_test = feats_test / np.std(feats_test, axis=2, keepdims=True)
    

    nn_model = get_model(feats_train.shape[1:])
    nn_model.compile(optimizer=optimizers.Adam(lr=0.001),
                     loss="sparse_categorical_crossentropy",
                     metrics=["accuracy"])
    
    cb = [ModelCheckpoint(save_prefix+"best_val_acc_model.h5", monitor="val_acc", save_best_only=True)
          , ModelCheckpoint(save_prefix+"best_val_loss_model.h5", monitor="val_loss", save_best_only=True)
          , ReduceLROnPlateau(patience=10, monitor="val_loss", verbose=1, factor=0.1, min_lr = 0.00005)
#           , EarlyStopping(monitor="val_loss", patience=30)
         ]
    
#     print(nn_model.summary())

    nn_model.fit(x=feats_train, y=labels_train,
                 validation_data = (feats_validation, labels_validation), 
                 callbacks=cb,  batch_size=64, epochs=200, verbose=1)
    
    print("Loading, validating and testing best model")
    nn_model = load_model(save_prefix+"best_val_loss_model.h5")
    print("Validation results:", nn_model.evaluate(feats_validation, labels_validation))
    
    # Predict the validation data
    validation_preds = nn_model.predict(feats_validation, batch_size=128)
    # Predict the test data
    test_preds = nn_model.predict(feats_test, batch_size=128)
    
    return test_preds, validation_preds

test_p, val_p = train_model(train_feats_embedding, train_labels, 
                            validation_feats_embedding, validation_labels, 
                            test_feats_embedding, embedding_weights, normalize_feats=False)

best_model = load_model("best_val_acc_model.h5")
val_preds = best_model.predict(validation_feats_embedding)
report_metrics(validation_labels, np.argmax(val_preds, axis=-1))

def report_metrics(true_labels, pred_labels):
#   classes_index = [CLASSES.index(c) for c in CLASSES]

  accuracy = metrics.accuracy_score(true_labels, pred_labels)
  macro_precision = metrics.precision_score(true_labels, pred_labels, average="macro")
  macro_recall = metrics.recall_score(true_labels, pred_labels, average="macro")
  macro_f1 = metrics.f1_score(true_labels, pred_labels, average="macro")
  micro_f1 = metrics.f1_score(true_labels, pred_labels, average="micro")
  
  return (accuracy, macro_precision, macro_recall, macro_f1, micro_f1)

def test_model(model, in_data, in_labels):
  # Predict the data
  pred_scores = model.predict(in_data, batch_size=128)
  y_classified_labels = np.argmax(pred_scores, axis=-1)
  
  return report_metrics(in_labels, y_classified_labels)

# Testeamos la validación
m = load_model("best_val_acc_model.h5")
accuracy, precision, recall, f1_macro, f1_micro = test_model(m, validation_feats_embedding, validation_labels)
print("Accuracy: %f" % accuracy)
print("Precision: %f" % precision)
print("Recall: %f" % recall)
print("F1-Macro: %f" % f1_macro)
print("F1-Micro: %f" % f1_micro)

def train_model_crossval(feats_train, labels_train, feats_validation, labels_validation, feats_test, word_embedding_weights, random_state=None):
    """Classification with RNN and embedings (pre-trained)
    """

    test_pred_list = []
    validation_pred_list = []
    validation_true_list = []
    
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state = random_state)
    
    all_feats = np.concatenate([feats_train,feats_validation], axis=0)
    all_labels = np.concatenate([labels_train, labels_validation], axis=0)
    for ii, (train_idx, validation_idx) in enumerate(skf.split(all_feats, all_labels)):
      print("Fold", ii)
      train_x = all_feats[train_idx]
      train_y = all_labels[train_idx]
      
      validation_x = all_feats[validation_idx]
      validation_y = all_labels[validation_idx]
      
      fold_test_preds, fold_validation_preds = train_model(train_x, train_y,
                                                           validation_x, validation_y,
                                                           feats_test, word_embedding_weights,
                                                           save_prefix="fold_"+str(ii)+"_",
                                                           normalize_feats=False, random_state=ii)
      
      validation_pred_list.append(fold_validation_preds)
      validation_true_list.append(validation_y)
      test_pred_list.append(fold_test_preds)
      
    return test_pred_list, validation_pred_list, validation_true_list

test_preds, validation_preds, validation_true_labels = train_model_crossval(train_feats_embedding, train_labels,
                                                                            validation_feats_embedding, validation_labels, 
                                                                            test_feats_embedding, embedding_weights, random_state = 42)

all_val_preds = np.concatenate([v for v in validation_preds], axis=0)
all_val_true_labels = np.concatenate([v for v in validation_true_labels], axis=0)
report_metrics(all_val_true_labels, np.argmax(all_val_preds, axis=-1))

aggregated_test_preds = np.array(test_preds)
mean_test_preds = np.mean(aggregated_test_preds, axis=0)
mean_test_labels = np.argmax(mean_test_preds, axis=-1)

# Sacamos las predicciones de test medias
aggregated_test_preds = np.array(test_preds)
mean_test_preds = np.mean(aggregated_test_preds, axis=0)
mean_test_labels = np.argmax(mean_test_preds, axis=-1)
    

pred_test_labels = np.array([CLASSES_inv[l] for l in mean_test_labels])
test_tweetid = np.array(test_df['tweetid'].values, dtype='int64')
# Cambio respecto a la predicción anterior
print("Cambios de etiquetas:", np.sum(pred_test_labels != prev_pred_test_labels))

pred_test_labels = np.array([CLASSES_inv[l] for l in np.argmax(test_p, axis=-1)])
test_tweetid = np.array(test_df['tweetid'].values, dtype='int64')
# Cambio respecto a la predicción anterior
print("Cambios de etiquetas:", np.sum(pred_test_labels != prev_pred_test_labels))

with open("/content/prediccion_nn.csv", 'w') as f:
  print("Id,Expected", file=f)
  for ii in np.arange(pred_test_labels.shape[0], dtype='int64'):
    print("%i,%s" % (test_tweetid[ii],pred_test_labels[ii]), file=f)
prev_pred_test_labels = pred_test_labels.copy()

ii=0
print(test_df.content.values[ii], pred_test_labels[ii])


# -*- coding: utf-8 -*-
"""Copy of FC_blocks

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1ulQthlWW_8ZFMprtaZeroWRlO1YIekpl
"""

import sys
import numpy as np
import re
from numpy.random import rand as np_rand
from numpy.random import seed as np_seed
from numpy import array as np_array
from nltk.tokenize.casual import TweetTokenizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn import metrics

from keras.models import Sequential, Model, load_model
from keras.layers import LSTM, GRU, CuDNNLSTM, CuDNNGRU
from keras.layers import BatchNormalization
from keras import optimizers
from keras.layers.core import Dense, Activation, Flatten, Reshape
from keras.layers import Add, Multiply
from keras.layers.core import Dropout
from keras.layers import Input, RepeatVector
from keras.layers.core import Flatten
from keras.layers import Bidirectional, TimeDistributed
from keras.preprocessing import sequence

from keras.layers.embeddings import Embedding
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping

from tqdm import tqdm



"""#Funciones"""

def read_embeddings(path, offset, random_state=42):
    """Load embeddings file.
    """
    word_embeddings = [[] for i in range(offset)]
    word_indexes = {}
    with open(path, "r", encoding="utf-8") as emb_file:
        emb_file.readline()
        for line in emb_file:
            fields = line.partition(EMB_SEP_CHAR)
            word = fields[0].strip()
            own_strip = str.strip
            emb_values = np_array([float(x) for x in own_strip(fields[-1]).split(EMB_SEP_CHAR)])
            word_indexes[word] = len(word_embeddings)
            word_embeddings.append(emb_values)
            
    # Offset = 2; Padding and OOV.
    np_seed(random_state)
    word_embeddings[0] = 2 * 0.1 * np_rand(len(word_embeddings[2])) - 1
    word_embeddings[1] = 2 * 0.1 * np_rand(len(word_embeddings[2])) - 1

    return (word_embeddings, word_indexes)

def tokenize(text):
    """Tokenize an input text
    
    Args:
        text: A String with the text to tokenize
    
    Returns:
        A list of Strings (tokens)
    """
    text_tokenized = TWEET_TOKENIZER.tokenize(text)
    return text_tokenized

def fit_transform_vocabulary_pretrain_embeddings(corpus, pre_embeddings_index):
    """Creates the vocabulary of the corpus.
        Index 0: padding
        Index 1: OOV.
    
    Args:
        corpus: A list os str (documents)
        
    Returns:
        A tuple whose first element is a dictionary word-index and the second
        element is a list of list in which each position is the index of the 
        token in the vocabulary
    """
    
    vocabulary = {}
    corpus_indexes = []
    corpus_indexes_append = corpus_indexes.append
    index = 0
    own_lowercase = str.lower
    for doc in tqdm(corpus, "Fitting tokenizer"):
        doc_indexes = []
        tokens = tokenize(own_lowercase(doc))
        for token in tokens:
            if RE_TOKEN_USER.fullmatch(token):
                token = "@user"
            if token in pre_embeddings_index:
                index = pre_embeddings_index[token]
                doc_indexes.append(index)
                if token not in vocabulary:
                    vocabulary[token] = index
            else:
                index = 1
                doc_indexes.append(index)
                if token not in vocabulary:
                    vocabulary[token] = index
        corpus_indexes_append(doc_indexes)
        
    return (vocabulary, corpus_indexes)

def calculate_quality_performance(y_labels, y_classified_labels, model_name):
    classes_index = [CLASSES.index(c) for c in CLASSES]
    accruacy = metrics.accuracy_score(y_labels, y_classified_labels)
    macro_precision = metrics.precision_score(y_labels, y_classified_labels, labels=classes_index, average="macro")
    macro_recall = metrics.recall_score(y_labels, y_classified_labels, labels=classes_index, average="macro")
    macro_f1 = metrics.f1_score(y_labels, y_classified_labels, labels=classes_index, average="macro")
    
    print("\n*** Results " + model_name + "***")
    print("Macro-Precision: " + str(macro_precision))
    print("Macro-Recall: " + str(macro_recall))
    print("Macro-F1: " + str(macro_f1))
    print("Accuracy: " + str(accruacy))

from google.colab import drive
drive.mount('/content/gdrive')

TWEET_TOKENIZER = TweetTokenizer(preserve_case=False, reduce_len=True, strip_handles=False)
CLASSES = []
EMB_SEP_CHAR = ' '
RE_TOKEN_USER = re.compile(r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){20}(?!@))|(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){1,19})(?![A-Za-z0-9_]*@)")


CLASSES = {'N':0, 'P':1, 'NEU':2, 'NONE':3}
CLASSES_inv = {v:k for k,v in CLASSES.items()}

# word2vec_path = '/content/fasttext_spanish_twitter_100d.vec'
word2vec_path = '/content/gdrive/My Drive/Colab Notebooks/fasttext_spanish_twitter_100d.vec'

import requests
import xml.etree.ElementTree as ET
import pandas as pd

# Primero cargamos los datos (ya los he subido a GH)
train_tagged_url = 'https://raw.githubusercontent.com/FGonzalezLopez/DM_AA_SentAnalysisTwitter/master/intertass-ES-train-tagged.xml'
validation_tagged_url = 'https://raw.githubusercontent.com/FGonzalezLopez/DM_AA_SentAnalysisTwitter/master/intertass-ES-development-tagged.xml'
test_url = 'https://raw.githubusercontent.com/FGonzalezLopez/DM_AA_SentAnalysisTwitter/master/intertass-ES-test.xml'

train_xml = ET.XML(requests.get(train_tagged_url).content)
val_xml = ET.XML(requests.get(validation_tagged_url).content)
test_xml = ET.XML(requests.get(test_url).content)

def parse_tweet(in_entry):
  # No sé si habrá una forma mejor (supongo que sí)
  vals_dict = dict()
  for attr in in_entry:
    if(not attr.tag == 'sentiment'):
      vals_dict[attr.tag] = attr.text
    else:
      vals_dict[attr.tag] = attr[0][0].text
          
  return vals_dict
  
train_df = pd.DataFrame.from_dict([parse_tweet(tweet) for tweet in tqdm(train_xml, "Parsing train tweets")])
val_df = pd.DataFrame.from_dict([parse_tweet(tweet) for tweet in tqdm(val_xml, "Parsing validation tweets")])
test_df = pd.DataFrame.from_dict([parse_tweet(tweet) for tweet in tqdm(test_xml, "Parsing test tweets")])

# Hacemos un shuffle (Importante!)
train_df = train_df.sample(frac=1.0).reset_index(drop=True)
val_df = val_df.sample(frac=1.0).reset_index(drop=True)

train_labels = train_df['sentiment'].values
tweets_train = train_df['content'].values
train_labels = np.array([CLASSES[c] for c in train_labels])

validation_labels = val_df['sentiment'].values
validation_labels = np.array([CLASSES[c] for c in validation_labels])
tweets_val = val_df['content'].values

tweets_test = test_df['content'].values

train_counts = np.unique(train_labels, return_counts=True)[1]
val_counts = np.unique(validation_labels, return_counts=True)[1]
print(train_counts/np.sum(train_counts))
print(val_counts/np.sum(val_counts))

def get_tokens(input_texts, word_index):
  corpus_tokens = []
  oov_tokens = 0
  for t in tqdm(input_texts, "Tokenize tweets"):
    tokens = tokenize(str.lower(t))
    t_indices = []
    for token in tokens:
      if(RE_TOKEN_USER.fullmatch(token) is not None):
        token = "@user"
      idx = word_index.get(token,1)
      if(idx == 1):
        oov_tokens+=1
      t_indices.append(idx)
    corpus_tokens.append(t_indices)
  return corpus_tokens, oov_tokens

def get_features(tweetsTrain, tweetsValidation, tweetsTest, embeddings_path, random_state=None):
    """Classification with RNN and embedings (pre-trained)
    """
    
    #Offset = 2; Padding and OOV.
    print("Begin loading embeddings.")
    word_embeddings, word_emb_indices = read_embeddings(embeddings_path, 2)
    print("End loading embeddings.")
    np_seed(random_state)
    
    
    #Build vocabulary and corpus indexes
    print("Computing training tokens")
    vocabulary_train, corpus_train_index = fit_transform_vocabulary_pretrain_embeddings(tweetsTrain, word_emb_indices)
    
    # Set a max input length
    max_len_input = int(np.percentile([len(tweet_train) for tweet_train in corpus_train_index], 95, axis=0))
    
    # Get the tokens for the validation and test sets
    print("Computing validation tokens")
    corpus_validation_index, oov_validation = get_tokens(tweetsValidation, word_emb_indices)
    print("OOV validation: %d" % oov_validation)
#     print(np.unique([el for sub in corpus_validation_index for el in sub], return_counts=True))
    
    print("Computing test tokens")
    corpus_test_index, oov_test = get_tokens(tweetsTest, word_emb_indices)
    print("OOV test: %d" % oov_test)
#     print(np.unique([el for sub in corpus_test_index for el in sub], return_counts=True))
    
    # Pad the train, validation and test token sequences
    train_features_pad = sequence.pad_sequences(corpus_train_index, maxlen=max_len_input, padding="post", truncating="post", dtype=type(corpus_train_index[0][0]))
    validation_features_pad = sequence.pad_sequences(corpus_validation_index, maxlen=max_len_input, padding="post", truncating="post", dtype=type(corpus_validation_index[0][0]))
    test_features_pad = sequence.pad_sequences(corpus_test_index, maxlen=max_len_input, padding="post", truncating="post", dtype=type(corpus_test_index[0][0]))
    
    return train_features_pad, validation_features_pad, test_features_pad, word_embeddings, word_emb_indices

def get_embedding_features(word_embedding_weights, in_data):
  # Separamos la extracción de características del embedding...
  embedding_model = Sequential()
  embedding_model.add((Embedding(len(word_embedding_weights), len(word_embedding_weights[0]), weights=[np_array(word_embedding_weights)], input_length=in_data.shape[1], trainable=False)))

  return embedding_model.predict(in_data, batch_size=256, verbose=1)

# Sacamos las features
train_feats, validation_feats, test_feats, embedding_weights, embedding_indices = get_features(tweets_train, tweets_val, tweets_test, word2vec_path, random_state=42)

train_feats_embedding = get_embedding_features(embedding_weights, train_feats)
validation_feats_embedding = get_embedding_features(embedding_weights, validation_feats)
test_feats_embedding = get_embedding_features(embedding_weights, test_feats)

def fc_module(in_tensor, units, drop_rate = 0.2, activation = 'elu', regularizer = 'l2'):
  out_tensor = in_tensor
  out_tensor = Dense(units, activation='linear', use_bias=False)(out_tensor)
  out_tensor = BatchNormalization(axis=-1)(out_tensor)
  out_tensor = Activation(activation)(out_tensor)
  out_tensor = Dropout(drop_rate)(out_tensor)
  return out_tensor

def get_model(in_data_shape):
  
  x_input = Input(shape = in_data_shape)
  
  x_output = x_input
  x_output = Dropout(0.4)(x_output)
  
#   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
  
#   # ATENSIÓ
#   attention_mask = fc_module(x_output, 10)
#   attention_mask = fc_module(x_output, 5)
#   attention_mask = Dense(1, kernel_regularizer='l2')(attention_mask)
#   attention_mask = Flatten()(attention_mask)
#   attention_mask = Activation('softmax')(attention_mask)
#   attention_mask = Reshape((-1,1))(attention_mask)
#   context = Multiply()([x_output, attention_mask])
  
#   x_output = Add()([x_output, context])
  
#   x_res = x_output
#   x_output = Dropout(0.2)(x_output)
#   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
#   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
#   x_output = Dropout(0.2)(x_output)
#   x_output = Add()([x_output, x_res])
  
  x_output = Bidirectional(CuDNNLSTM(10, return_sequences=False, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output)
  
  x_res = x_output
  x_output = Dropout(0.2)(x_output)
  x_output = fc_module(x_output, 20)
  x_output = Add()([x_output, x_res])
  x_output = Dropout(0.2)(x_output)
  x_output = fc_module(x_output, 20)
  x_res = x_output
  x_output = fc_module(x_output, 20)
  x_output = Add()([x_output, x_res])
  x_output = Dropout(0.2)(x_output)


  x_output = Dense(len(CLASSES), activation='softmax', kernel_regularizer='l2')(x_output)
  
  model = Model(inputs=x_input, outputs = x_output)
  
  return model

get_model((28,100)).summary()

def train_model(feats_train, labels_train, feats_validation, labels_validation, feats_test, word_embedding_weights, save_prefix = "", random_state=None, normalize_feats=True,):
    """Classification with RNN and embedings (pre-trained)
    """

    if(normalize_feats):
      # Normalizamos las features
      feats_train = feats_train - np.mean(feats_train, axis=2, keepdims=True)
      feats_validation = feats_validation - np.mean(feats_validation, axis=2, keepdims=True)
      feats_test = feats_test - np.mean(feats_test, axis=2, keepdims=True)

      feats_train = feats_train / np.std(feats_train, axis=2, keepdims=True)
      feats_validation = feats_validation / np.std(feats_validation, axis=2, keepdims=True)
      feats_test = feats_test / np.std(feats_test, axis=2, keepdims=True)
    

    nn_model = get_model(feats_train.shape[1:])
    nn_model.compile(optimizer=optimizers.Adam(lr=0.001),
                     loss="sparse_categorical_crossentropy",
                     metrics=["accuracy"])
    
    cb = [ModelCheckpoint(save_prefix+"best_val_acc_model.h5", monitor="val_acc", save_best_only=True)
          , ModelCheckpoint(save_prefix+"best_val_loss_model.h5", monitor="val_loss", save_best_only=True)
          , ReduceLROnPlateau(patience=10, monitor="val_loss", verbose=1, factor=0.1, min_lr = 0.00005)
#           , EarlyStopping(monitor="val_loss", patience=30)
         ]
    
#     print(nn_model.summary())

    nn_model.fit(x=feats_train, y=labels_train,
                 validation_data = (feats_validation, labels_validation), 
                 callbacks=cb,  batch_size=64, epochs=200, verbose=1)
    
    print("Loading, validating and testing best model")
    nn_model = load_model(save_prefix+"best_val_loss_model.h5")
    print("Validation results:", nn_model.evaluate(feats_validation, labels_validation))
    
    # Predict the validation data
    validation_preds = nn_model.predict(feats_validation, batch_size=128)
    # Predict the test data
    test_preds = nn_model.predict(feats_test, batch_size=128)
    
    return test_preds, validation_preds

test_p, val_p = train_model(train_feats_embedding, train_labels, 
                            validation_feats_embedding, validation_labels, 
                            test_feats_embedding, embedding_weights, normalize_feats=False)

best_model = load_model("best_val_acc_model.h5")
val_preds = best_model.predict(validation_feats_embedding)
report_metrics(validation_labels, np.argmax(val_preds, axis=-1))

def report_metrics(true_labels, pred_labels):
#   classes_index = [CLASSES.index(c) for c in CLASSES]

  accuracy = metrics.accuracy_score(true_labels, pred_labels)
  macro_precision = metrics.precision_score(true_labels, pred_labels, average="macro")
  macro_recall = metrics.recall_score(true_labels, pred_labels, average="macro")
  macro_f1 = metrics.f1_score(true_labels, pred_labels, average="macro")
  micro_f1 = metrics.f1_score(true_labels, pred_labels, average="micro")
  
  return (accuracy, macro_precision, macro_recall, macro_f1, micro_f1)

def test_model(model, in_data, in_labels):
  # Predict the data
  pred_scores = model.predict(in_data, batch_size=128)
  y_classified_labels = np.argmax(pred_scores, axis=-1)
  
  return report_metrics(in_labels, y_classified_labels)

# Testeamos la validación
m = load_model("best_val_acc_model.h5")
accuracy, precision, recall, f1_macro, f1_micro = test_model(m, validation_feats_embedding, validation_labels)
print("Accuracy: %f" % accuracy)
print("Precision: %f" % precision)
print("Recall: %f" % recall)
print("F1-Macro: %f" % f1_macro)
print("F1-Micro: %f" % f1_micro)

def train_model_crossval(feats_train, labels_train, feats_validation, labels_validation, feats_test, word_embedding_weights, random_state=None):
    """Classification with RNN and embedings (pre-trained)
    """

    test_pred_list = []
    validation_pred_list = []
    validation_true_list = []
    
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state = random_state)
    
    all_feats = np.concatenate([feats_train,feats_validation], axis=0)
    all_labels = np.concatenate([labels_train, labels_validation], axis=0)
    for ii, (train_idx, validation_idx) in enumerate(skf.split(all_feats, all_labels)):
      print("Fold", ii)
      train_x = all_feats[train_idx]
      train_y = all_labels[train_idx]
      
      validation_x = all_feats[validation_idx]
      validation_y = all_labels[validation_idx]
      
      fold_test_preds, fold_validation_preds = train_model(train_x, train_y,
                                                           validation_x, validation_y,
                                                           feats_test, word_embedding_weights,
                                                           save_prefix="fold_"+str(ii)+"_",
                                                           normalize_feats=False, random_state=ii)
      
      validation_pred_list.append(fold_validation_preds)
      validation_true_list.append(validation_y)
      test_pred_list.append(fold_test_preds)
      
    return test_pred_list, validation_pred_list, validation_true_list

test_preds, validation_preds, validation_true_labels = train_model_crossval(train_feats_embedding, train_labels,
                                                                            validation_feats_embedding, validation_labels, 
                                                                            test_feats_embedding, embedding_weights, random_state = 42)

all_val_preds = np.concatenate([v for v in validation_preds], axis=0)
all_val_true_labels = np.concatenate([v for v in validation_true_labels], axis=0)
report_metrics(all_val_true_labels, np.argmax(all_val_preds, axis=-1))

aggregated_test_preds = np.array(test_preds)
mean_test_preds = np.mean(aggregated_test_preds, axis=0)
mean_test_labels = np.argmax(mean_test_preds, axis=-1)

# Sacamos las predicciones de test medias
aggregated_test_preds = np.array(test_preds)
mean_test_preds = np.mean(aggregated_test_preds, axis=0)
mean_test_labels = np.argmax(mean_test_preds, axis=-1)
    

pred_test_labels = np.array([CLASSES_inv[l] for l in mean_test_labels])
test_tweetid = np.array(test_df['tweetid'].values, dtype='int64')
# Cambio respecto a la predicción anterior
print("Cambios de etiquetas:", np.sum(pred_test_labels != prev_pred_test_labels))

pred_test_labels = np.array([CLASSES_inv[l] for l in np.argmax(test_p, axis=-1)])
test_tweetid = np.array(test_df['tweetid'].values, dtype='int64')
# Cambio respecto a la predicción anterior
print("Cambios de etiquetas:", np.sum(pred_test_labels != prev_pred_test_labels))

with open("/content/prediccion_nn.csv", 'w') as f:
  print("Id,Expected", file=f)
  for ii in np.arange(pred_test_labels.shape[0], dtype='int64'):
    print("%i,%s" % (test_tweetid[ii],pred_test_labels[ii]), file=f)
prev_pred_test_labels = pred_test_labels.copy()

ii=0
print(test_df.content.values[ii], pred_test_labels[ii])


# -*- coding: utf-8 -*- Copy of FC_blocks  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1ulQthlWW_8ZFMprtaZeroWRlO1YIekpl #Funciones Load embeddings file. # Offset = 2; Padding and OOV. Tokenize an input text          Args:         text: A String with the text to tokenize          Returns:         A list of Strings (tokens) Creates the vocabulary of the corpus.         Index 0: padding         Index 1: OOV.          Args:         corpus: A list os str (documents)              Returns:         A tuple whose first element is a dictionary word-index and the second         element is a list of list in which each position is the index of the          token in the vocabulary #\$%&*])@(([A-Za-z0-9_]){20}(?!@))|(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){1,19})(?![A-Za-z0-9_]*@)") # word2vec_path = '/content/fasttext_spanish_twitter_100d.vec' # Primero cargamos los datos (ya los he subido a GH) # No sé si habrá una forma mejor (supongo que sí) # Hacemos un shuffle (Importante!) Classification with RNN and embedings (pre-trained) #Offset = 2; Padding and OOV. #Build vocabulary and corpus indexes # Set a max input length # Get the tokens for the validation and test sets #     print(np.unique([el for sub in corpus_validation_index for el in sub], return_counts=True)) #     print(np.unique([el for sub in corpus_test_index for el in sub], return_counts=True)) # Pad the train, validation and test token sequences # Separamos la extracción de características del embedding... # Sacamos las features #   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output) #   # ATENSIÓ #   attention_mask = fc_module(x_output, 10) #   attention_mask = fc_module(x_output, 5) #   attention_mask = Dense(1, kernel_regularizer='l2')(attention_mask) #   attention_mask = Flatten()(attention_mask) #   attention_mask = Activation('softmax')(attention_mask) #   attention_mask = Reshape((-1,1))(attention_mask) #   context = Multiply()([x_output, attention_mask]) #   x_output = Add()([x_output, context]) #   x_res = x_output #   x_output = Dropout(0.2)(x_output) #   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output) #   x_output = Bidirectional(CuDNNLSTM(10, return_sequences=True, kernel_regularizer='l2', recurrent_regularizer='l2'))(x_output) #   x_output = Dropout(0.2)(x_output) #   x_output = Add()([x_output, x_res]) Classification with RNN and embedings (pre-trained) # Normalizamos las features #           , EarlyStopping(monitor="val_loss", patience=30) #     print(nn_model.summary()) # Predict the validation data # Predict the test data #   classes_index = [CLASSES.index(c) for c in CLASSES] # Predict the data # Testeamos la validación Classification with RNN and embedings (pre-trained) # Sacamos las predicciones de test medias # Cambio respecto a la predicción anterior # Cambio respecto a la predicción anterior