#This is CNN + LSTM, with SMOTE oversampling

import numpy
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Conv1D, MaxPooling1D, Dropout, Dense, Flatten
from keras.layers import LSTM, Flatten
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
import codecs
import nltk
from nltk.corpus import stopwords
from os import listdir
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import f1_score
from sklearn.utils import class_weight
import numpy as np
import random
from keras.models import model_from_json
import h5py
from keras.preprocessing.text import one_hot
from keras.preprocessing.sequence import pad_sequences
from imblearn.over_sampling import SMOTE
from sklearn.model_selection import train_test_split



def try_lstm(X, Y):
	numpy.random.seed(7)
	top_words = 5000
	max_review_length = 150
	
	sm = SMOTE(ratio = 'minority', random_state=1, kind='regular', k_neighbors = 5)

	encoded_sentences = [one_hot(d, top_words) for d in X]
        padded_sentences = pad_sequences(encoded_sentences, maxlen = max_review_length, padding='post')
	
	X_res, Y_res = sm.fit_sample(padded_sentences, Y)
	X_train, X_test, y_train, y_test = train_test_split(X_res, Y_res, test_size = 0.33, random_state = 1, shuffle = True)

	myclass_weight = class_weight.compute_class_weight('balanced', np.unique(y_train), y_train)
	class_weight_dict = dict(enumerate(myclass_weight))
	# create the model
	embedding_vecor_length = 100
	model = Sequential()
	model.add(Embedding(top_words, embedding_vecor_length, input_length=max_review_length))
	model.add(Conv1D(128, 5, activation='relu', padding='same'))
	model.add(Conv1D(128, 5, activation='relu', padding='same'))
	model.add(MaxPooling1D(5))
	model.add(Dropout(0.5))
	model.add(Dense(1, activation='sigmoid'))
	#input=model.layers[7].output.get_shape(),
	model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2, name='lstm1', return_sequences=True))
	model.add(LSTM(32, dropout=0.2, recurrent_dropout=0.2, name='lstm2', return_sequences=True))
	model.add(Flatten())
	model.add(Dense(128, activation='relu', name='dense1'))
	model.add(Dropout(0.5))
	model.add(Dense(1, activation='sigmoid'))
	model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
	print(model.summary())
	model.fit(X_train, y_train, epochs=3, batch_size=64, class_weight=class_weight_dict)
	# Final evaluation of the model
	scores = model.evaluate(X_test, y_test, verbose=0)

	a = model.predict_classes(X_test, verbose=1)
	print(precision_recall_fscore_support(y_test, a))
	print('F-score :', f1_score(y_test, a))	
	print("Accuracy: %.2f%%" % (scores[1]*100))

def generateData(dataFolder):
	stops = set(stopwords.words("english"))  
	X = []
	Y = []
	sentence = ''
	relevance = 0
	for fileName in listdir(dataFolder):
		f = codecs.open(dataFolder + fileName, encoding='utf-8')
		for line in f:
			if '\n' in line and len(line) == 2: #End of a Sentence
				if sentence != '':
					X.append(sentence)
					Y.append(relevance)
				#Reset sentence and relevance for next line
				sentence = ''
				relevance = 0 
			else:	#Same sentence still being continued
				token = line.split(' ')[0].lower()
				#Keep only words of letters and not stop words
				if token not in stops and token not in ",!.:<>":
					if sentence == '': 
						sentence = token
					else: 
						sentence += ' ' + token
				if line[:-1].split(' ')[-1] != 'O': #if it is annotated, then it is relevant 
					relevance = 1

	return X,Y
#Get the training data
X, Y = generateData("training_material/data/tokenized/")

try_lstm(X, Y)

#This is CNN + LSTM, with SMOTE oversampling

import numpy
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Conv1D, MaxPooling1D, Dropout, Dense, Flatten
from keras.layers import LSTM, Flatten
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
import codecs
import nltk
from nltk.corpus import stopwords
from os import listdir
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import f1_score
from sklearn.utils import class_weight
import numpy as np
import random
from keras.models import model_from_json
import h5py
from keras.preprocessing.text import one_hot
from keras.preprocessing.sequence import pad_sequences
from imblearn.over_sampling import SMOTE
from sklearn.model_selection import train_test_split



def try_lstm(X, Y):
	numpy.random.seed(7)
	top_words = 5000
	max_review_length = 150
	
	sm = SMOTE(ratio = 'minority', random_state=1, kind='regular', k_neighbors = 5)

	encoded_sentences = [one_hot(d, top_words) for d in X]
        padded_sentences = pad_sequences(encoded_sentences, maxlen = max_review_length, padding='post')
	
	X_res, Y_res = sm.fit_sample(padded_sentences, Y)
	X_train, X_test, y_train, y_test = train_test_split(X_res, Y_res, test_size = 0.33, random_state = 1, shuffle = True)

	myclass_weight = class_weight.compute_class_weight('balanced', np.unique(y_train), y_train)
	class_weight_dict = dict(enumerate(myclass_weight))
	# create the model
	embedding_vecor_length = 100
	model = Sequential()
	model.add(Embedding(top_words, embedding_vecor_length, input_length=max_review_length))
	model.add(Conv1D(128, 5, activation='relu', padding='same'))
	model.add(Conv1D(128, 5, activation='relu', padding='same'))
	model.add(MaxPooling1D(5))
	model.add(Dropout(0.5))
	model.add(Dense(1, activation='sigmoid'))
	#input=model.layers[7].output.get_shape(),
	model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2, name='lstm1', return_sequences=True))
	model.add(LSTM(32, dropout=0.2, recurrent_dropout=0.2, name='lstm2', return_sequences=True))
	model.add(Flatten())
	model.add(Dense(128, activation='relu', name='dense1'))
	model.add(Dropout(0.5))
	model.add(Dense(1, activation='sigmoid'))
	model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
	print(model.summary())
	model.fit(X_train, y_train, epochs=3, batch_size=64, class_weight=class_weight_dict)
	# Final evaluation of the model
	scores = model.evaluate(X_test, y_test, verbose=0)

	a = model.predict_classes(X_test, verbose=1)
	print(precision_recall_fscore_support(y_test, a))
	print('F-score :', f1_score(y_test, a))	
	print("Accuracy: %.2f%%" % (scores[1]*100))

def generateData(dataFolder):
	stops = set(stopwords.words("english"))  
	X = []
	Y = []
	sentence = ''
	relevance = 0
	for fileName in listdir(dataFolder):
		f = codecs.open(dataFolder + fileName, encoding='utf-8')
		for line in f:
			if '\n' in line and len(line) == 2: #End of a Sentence
				if sentence != '':
					X.append(sentence)
					Y.append(relevance)
				#Reset sentence and relevance for next line
				sentence = ''
				relevance = 0 
			else:	#Same sentence still being continued
				token = line.split(' ')[0].lower()
				#Keep only words of letters and not stop words
				if token not in stops and token not in ",!.:<>":
					if sentence == '': 
						sentence = token
					else: 
						sentence += ' ' + token
				if line[:-1].split(' ')[-1] != 'O': #if it is annotated, then it is relevant 
					relevance = 1

	return X,Y
#Get the training data
X, Y = generateData("training_material/data/tokenized/")

try_lstm(X, Y)

#This is CNN + LSTM, with SMOTE oversampling # create the model #input=model.layers[7].output.get_shape(), # Final evaluation of the model #End of a Sentence #Reset sentence and relevance for next line #Same sentence still being continued #Keep only words of letters and not stop words #if it is annotated, then it is relevant #Get the training data