from summariser.utils.corpus_reader import CorpusReader
from resources import PROCESSED_PATH, , W2V_PATH, FEATURE_DIR, LANGUAGE
from summariser.utils.reader import readSummaries
from nltk.corpus import stopwords
from summariser.utils.misc import cosine
from summariser.utils.data_helpers import sent2tokens_wostop
import os
from gensim.models import KeyedVectors, LdaModel
from gensim.models.keyedvectors import WordEmbeddingsKeyedVectors
import numpy as np
from collections import OrderedDict

class WordEmbeddingRewarder():

    def __init__(self):
        self.word_embedding = KeyedVectors.load_word2vec_format(W2V_PATH)

    def __call__(self, docs, summaries):
        tfs = []
        df = OrderedDict()
        weights = []
        entities = []
        for doc in docs:
            tf = OrderedDict()
            token_found = set()
            doc_token = []
            for sent in doc[1]:
                sent = sent2tokens_wostop(sent, set(stopwords.words(LANGUAGE)), LANGUAGE)
                for token in sent:
                    if token in tf:
                        tf[token] += 1
                    else:
                        tf[token] = 1
                    if token not in token_found:
                        token_found.add(token)
                        if token in df:
                            df[token] += 1
                        else:
                            df[token] = 1
                    embedding = np.zeros(300)
                    try:
                        embedding += self.word_embedding[token]
                    except KeyError:
                        pass
                embedding /= len(embedding)
                weights.append(embedding)
                entities.append(str(len(entities)))
            tfs.append(tf)

        id2word = {i:word for i, word in enumerate(df.keys())}
        word2id = {id2word[id]:id for id in id2word.keys()}
        corpora = [[(word2id[token], tf[token]) for token in tf.keys()] for tf in tfs]
        self.doc_entities = []
        for i, tf in enumerate(tfs):
            divisor = sum([tf[token]/df[token] for token in tf.keys()])
            embedding = []
            for token in tf.keys():
                try:
                    embedding.append(self.word_embedding[token]*tf[token]/df[token])
                except KeyError:
                    pass
            embedding = np.sum(np.array(embedding), 0)/(len(embedding)*divisor)
            weights.append(embedding)
            entities.append('d'+str(i))
            self.doc_entities.append('d'+str(i))

        self.lda = LdaModel(corpus=corpora, num_topics=10, id2word=id2word, passes=10)

        self.topic_entities = []
        for i in range(10):
            topic_words = self.lda.show_topic(i, topn=30)
            embedding = []
            divisor = sum([w_p_pair[1]for w_p_pair in topic_words])
            for w_p_pair in topic_words:
                try:
                    embedding.append(self.word_embedding[w_p_pair[0]]*w_p_pair[1]/divisor)
                except KeyError:
                    pass
            embedding = np.sum(np.array(embedding), 0)/len(embedding)
            weights.append(embedding)
            entities.append('t'+str(i))
            self.topic_entities.append('t'+str(i))
        self.sent_embedding = WordEmbeddingsKeyedVectors(300)
        self.sent_embedding.add(entities, np.array(weights), replace=True)

        return self.distributional_semantic_similarity(summaries), self.topic_relevance(summaries), self.coherence(summaries)

    def distributional_semantic_similarity(self, summaries):
        results = []
        for summ in summaries:
            sent_entities = list(map(str, summ))
            wmd = self.sent_embedding.wmdistance(sent_entities , self.doc_entities)
            results.append(wmd)
        return results

    def topic_relevance(self, summaries):
        results = []
        for summ in summaries:
            sent_entities = list(map(str, summ))
            wmd = self.sent_embedding.wmdistance(sent_entities , self.topic_entities)
            results.append(wmd)
        return results

    def coherence(self, summaries):
        results = []
        for summ in summaries:
            sim = []
            for i in range(len(summ)-1):
                s = cosine(self.sent_embedding[str(summ[i])], self.sent_embedding[str(summ[i+1])])
                if not np.isnan(s):
                    sim.append(s)
            sim = np.array(sim)
            results.append([np.mean(sim), np.std(sim)])
        return results

if __name__ == '__main__':
    dataset = 'DUC2001' ## DUC2001, DUC2002, DUC2004
    sample_num = 9999
    out_base = os.path.join(FEATURE_DIR,dataset)
    if not os.path.exists(out_base):
        os.makedirs(out_base)

    ### read documents and ref. summaries
    reader = CorpusReader(PROCESSED_PATH)
    data = reader.get_data(dataset)

    ### read data
    rewarder = WordEmbeddingRewarder()
    for topic,docs,models in data:

        print('read DATA {}, TOPIC {}'.format(dataset,topic))
        summs, _ = readSummaries(dataset,topic,'rouge',sample_num)
        values = rewarder(docs, summs)
        assert len(values) == len(summs)

        out_str = ''
        for ii,vv in enumerate(values[0]):
            out_str += '{}\t{}\n'.format(summs[ii],vv)

        if not os.path.exists(os.path.join(out_base,topic)):
            os.makedirs(os.path.join(out_base,topic))

        fpath = os.path.join(out_base,topic,'dss')
        ff = open(fpath,'w')
        ff.write(out_str)
        ff.close()

        out_str = ''
        for ii,vv in enumerate(values[1]):
            out_str += '{}\t{}\n'.format(summs[ii],vv)

        if not os.path.exists(os.path.join(out_base,topic)):
            os.makedirs(os.path.join(out_base,topic))

        fpath = os.path.join(out_base,topic,'topic_rel')
        ff = open(fpath,'w')
        ff.write(out_str)
        ff.close()

        out_str = ''
        for ii,vv in enumerate(values[2]):
            out_str += '{}\t{}\t{}\n'.format(summs[ii],vv[0], vv[1])

        if not os.path.exists(os.path.join(out_base,topic)):
            os.makedirs(os.path.join(out_base,topic))

        fpath = os.path.join(out_base,topic,'coherence_mean_std')
        ff = open(fpath,'w')
        ff.write(out_str)
        ff.close()



from summariser.utils.corpus_reader import CorpusReader
from resources import PROCESSED_PATH, , W2V_PATH, FEATURE_DIR, LANGUAGE
from summariser.utils.reader import readSummaries
from nltk.corpus import stopwords
from summariser.utils.misc import cosine
from summariser.utils.data_helpers import sent2tokens_wostop
import os
from gensim.models import KeyedVectors, LdaModel
from gensim.models.keyedvectors import WordEmbeddingsKeyedVectors
import numpy as np
from collections import OrderedDict

class WordEmbeddingRewarder():

    def __init__(self):
        self.word_embedding = KeyedVectors.load_word2vec_format(W2V_PATH)

    def __call__(self, docs, summaries):
        tfs = []
        df = OrderedDict()
        weights = []
        entities = []
        for doc in docs:
            tf = OrderedDict()
            token_found = set()
            doc_token = []
            for sent in doc[1]:
                sent = sent2tokens_wostop(sent, set(stopwords.words(LANGUAGE)), LANGUAGE)
                for token in sent:
                    if token in tf:
                        tf[token] += 1
                    else:
                        tf[token] = 1
                    if token not in token_found:
                        token_found.add(token)
                        if token in df:
                            df[token] += 1
                        else:
                            df[token] = 1
                    embedding = np.zeros(300)
                    try:
                        embedding += self.word_embedding[token]
                    except KeyError:
                        pass
                embedding /= len(embedding)
                weights.append(embedding)
                entities.append(str(len(entities)))
            tfs.append(tf)

        id2word = {i:word for i, word in enumerate(df.keys())}
        word2id = {id2word[id]:id for id in id2word.keys()}
        corpora = [[(word2id[token], tf[token]) for token in tf.keys()] for tf in tfs]
        self.doc_entities = []
        for i, tf in enumerate(tfs):
            divisor = sum([tf[token]/df[token] for token in tf.keys()])
            embedding = []
            for token in tf.keys():
                try:
                    embedding.append(self.word_embedding[token]*tf[token]/df[token])
                except KeyError:
                    pass
            embedding = np.sum(np.array(embedding), 0)/(len(embedding)*divisor)
            weights.append(embedding)
            entities.append('d'+str(i))
            self.doc_entities.append('d'+str(i))

        self.lda = LdaModel(corpus=corpora, num_topics=10, id2word=id2word, passes=10)

        self.topic_entities = []
        for i in range(10):
            topic_words = self.lda.show_topic(i, topn=30)
            embedding = []
            divisor = sum([w_p_pair[1]for w_p_pair in topic_words])
            for w_p_pair in topic_words:
                try:
                    embedding.append(self.word_embedding[w_p_pair[0]]*w_p_pair[1]/divisor)
                except KeyError:
                    pass
            embedding = np.sum(np.array(embedding), 0)/len(embedding)
            weights.append(embedding)
            entities.append('t'+str(i))
            self.topic_entities.append('t'+str(i))
        self.sent_embedding = WordEmbeddingsKeyedVectors(300)
        self.sent_embedding.add(entities, np.array(weights), replace=True)

        return self.distributional_semantic_similarity(summaries), self.topic_relevance(summaries), self.coherence(summaries)

    def distributional_semantic_similarity(self, summaries):
        results = []
        for summ in summaries:
            sent_entities = list(map(str, summ))
            wmd = self.sent_embedding.wmdistance(sent_entities , self.doc_entities)
            results.append(wmd)
        return results

    def topic_relevance(self, summaries):
        results = []
        for summ in summaries:
            sent_entities = list(map(str, summ))
            wmd = self.sent_embedding.wmdistance(sent_entities , self.topic_entities)
            results.append(wmd)
        return results

    def coherence(self, summaries):
        results = []
        for summ in summaries:
            sim = []
            for i in range(len(summ)-1):
                s = cosine(self.sent_embedding[str(summ[i])], self.sent_embedding[str(summ[i+1])])
                if not np.isnan(s):
                    sim.append(s)
            sim = np.array(sim)
            results.append([np.mean(sim), np.std(sim)])
        return results

if __name__ == '__main__':
    dataset = 'DUC2001' ## DUC2001, DUC2002, DUC2004
    sample_num = 9999
    out_base = os.path.join(FEATURE_DIR,dataset)
    if not os.path.exists(out_base):
        os.makedirs(out_base)

    ### read documents and ref. summaries
    reader = CorpusReader(PROCESSED_PATH)
    data = reader.get_data(dataset)

    ### read data
    rewarder = WordEmbeddingRewarder()
    for topic,docs,models in data:

        print('read DATA {}, TOPIC {}'.format(dataset,topic))
        summs, _ = readSummaries(dataset,topic,'rouge',sample_num)
        values = rewarder(docs, summs)
        assert len(values) == len(summs)

        out_str = ''
        for ii,vv in enumerate(values[0]):
            out_str += '{}\t{}\n'.format(summs[ii],vv)

        if not os.path.exists(os.path.join(out_base,topic)):
            os.makedirs(os.path.join(out_base,topic))

        fpath = os.path.join(out_base,topic,'dss')
        ff = open(fpath,'w')
        ff.write(out_str)
        ff.close()

        out_str = ''
        for ii,vv in enumerate(values[1]):
            out_str += '{}\t{}\n'.format(summs[ii],vv)

        if not os.path.exists(os.path.join(out_base,topic)):
            os.makedirs(os.path.join(out_base,topic))

        fpath = os.path.join(out_base,topic,'topic_rel')
        ff = open(fpath,'w')
        ff.write(out_str)
        ff.close()

        out_str = ''
        for ii,vv in enumerate(values[2]):
            out_str += '{}\t{}\t{}\n'.format(summs[ii],vv[0], vv[1])

        if not os.path.exists(os.path.join(out_base,topic)):
            os.makedirs(os.path.join(out_base,topic))

        fpath = os.path.join(out_base,topic,'coherence_mean_std')
        ff = open(fpath,'w')
        ff.write(out_str)
        ff.close()



## DUC2001, DUC2002, DUC2004 ### read documents and ref. summaries ### read data