# -*- coding: utf-8 -*-
"""
Created on Sun Dec 13 00:21:22 2020

@author: lankuohsing
"""
"""
https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html#sklearn.feature_extraction.text.TfidfVectorizer.fit_transform
https://scikit-learn.org/stable/modules/feature_extraction.html#text-feature-extraction
"""
import math
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
import json
import string
from collections import defaultdict
# In[]

# In[]
"""" sklearn implementation """
def tf_idf_sklearn(corpus):
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(corpus)# L2 regularization y default
    # In[]
    #print(X)
    X_dense=X.todense()# sentence embeddings of corpus
    # In[]
    print(vectorizer.get_feature_names())
    print(vectorizer.vocabulary_)
    print(vectorizer.idf_)
    print(X_dense)
    # In[]
    dict_word_idf_sk={}
    for word, index in vectorizer.vocabulary_.items():
        dict_word_idf_sk[word]=vectorizer.idf_[index]
    # In[]

    with open('./models/dict_word_idf.json', 'w') as f:
        json.dump(dict_word_idf_sk, f)
    return X_dense
# In[]
"""  implementation by numpy and math """
def tf_idf_numpy(corpus):
    set_word=set()
    dict_word_numDoc=defaultdict(int)
    # In[]
    for i in range(0,len(corpus)):
        split_list=corpus[i][0:-1].split(" ")
        set_temp_word=set()
        for word in split_list:
            word=word.lower()
            set_word.add(word)
            if word in set_temp_word:
                continue
            dict_word_numDoc[word]+=1
            set_temp_word.add(word)
    set_word=list(set_word)
    set_word.sort()
    # In[]
    dict_word_idf={}
    for word in set_word:
        dict_word_idf[word]=math.log((1+len(corpus))/(1+dict_word_numDoc[word]))+1
    print(dict_word_idf)
    dict_word_index={}
    i=0
    for word in set_word:
        dict_word_index[word]=i
        i+=1
    # In[]
    sentence_vector=np.zeros((len(corpus),len(set_word)))

    for i in range(0,len(corpus)):
        word_list=corpus[i].lower().translate(str.maketrans('', '', string.punctuation)).split(" ")
        set_temp_word=set()
        dict_word_count={}
        for word in word_list:
            set_temp_word.add(word)
            if word in dict_word_count.keys():
                dict_word_count[word]+=1
            else:
                dict_word_count[word]=1
        for word in set_temp_word:
            sentence_vector[i][dict_word_index[word]]=dict_word_count[word]*dict_word_idf[word]

    # In[]
    sentence_vector_l2_norm=np.divide(sentence_vector,np.sqrt(np.sum(np.square(sentence_vector),1)).reshape(sentence_vector.shape[0],1))# sentence Embeddings of corpus
    return sentence_vector_l2_norm

# In[]


if __name__=="__main__":
    corpus = [
        'This is the first document.',
        'This document is the second document.',
        'And this is the third one.',
        'Is this the first document?',
    ]
    X_dense_1=tf_idf_sklearn(corpus)
    X_dense_2=tf_idf_numpy(corpus)
    print(X_dense_1-X_dense_2)# should be all zeros

# -*- coding: utf-8 -*-
"""
Created on Sun Dec 13 00:21:22 2020

@author: lankuohsing
"""
"""
https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html#sklearn.feature_extraction.text.TfidfVectorizer.fit_transform
https://scikit-learn.org/stable/modules/feature_extraction.html#text-feature-extraction
"""
import math
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
import json
import string
from collections import defaultdict
# In[]

# In[]
"""" sklearn implementation """
def tf_idf_sklearn(corpus):
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(corpus)# L2 regularization y default
    # In[]
    #print(X)
    X_dense=X.todense()# sentence embeddings of corpus
    # In[]
    print(vectorizer.get_feature_names())
    print(vectorizer.vocabulary_)
    print(vectorizer.idf_)
    print(X_dense)
    # In[]
    dict_word_idf_sk={}
    for word, index in vectorizer.vocabulary_.items():
        dict_word_idf_sk[word]=vectorizer.idf_[index]
    # In[]

    with open('./models/dict_word_idf.json', 'w') as f:
        json.dump(dict_word_idf_sk, f)
    return X_dense
# In[]
"""  implementation by numpy and math """
def tf_idf_numpy(corpus):
    set_word=set()
    dict_word_numDoc=defaultdict(int)
    # In[]
    for i in range(0,len(corpus)):
        split_list=corpus[i][0:-1].split(" ")
        set_temp_word=set()
        for word in split_list:
            word=word.lower()
            set_word.add(word)
            if word in set_temp_word:
                continue
            dict_word_numDoc[word]+=1
            set_temp_word.add(word)
    set_word=list(set_word)
    set_word.sort()
    # In[]
    dict_word_idf={}
    for word in set_word:
        dict_word_idf[word]=math.log((1+len(corpus))/(1+dict_word_numDoc[word]))+1
    print(dict_word_idf)
    dict_word_index={}
    i=0
    for word in set_word:
        dict_word_index[word]=i
        i+=1
    # In[]
    sentence_vector=np.zeros((len(corpus),len(set_word)))

    for i in range(0,len(corpus)):
        word_list=corpus[i].lower().translate(str.maketrans('', '', string.punctuation)).split(" ")
        set_temp_word=set()
        dict_word_count={}
        for word in word_list:
            set_temp_word.add(word)
            if word in dict_word_count.keys():
                dict_word_count[word]+=1
            else:
                dict_word_count[word]=1
        for word in set_temp_word:
            sentence_vector[i][dict_word_index[word]]=dict_word_count[word]*dict_word_idf[word]

    # In[]
    sentence_vector_l2_norm=np.divide(sentence_vector,np.sqrt(np.sum(np.square(sentence_vector),1)).reshape(sentence_vector.shape[0],1))# sentence Embeddings of corpus
    return sentence_vector_l2_norm

# In[]


if __name__=="__main__":
    corpus = [
        'This is the first document.',
        'This document is the second document.',
        'And this is the third one.',
        'Is this the first document?',
    ]
    X_dense_1=tf_idf_sklearn(corpus)
    X_dense_2=tf_idf_numpy(corpus)
    print(X_dense_1-X_dense_2)# should be all zeros

# -*- coding: utf-8 -*- Created on Sun Dec 13 00:21:22 2020  @author: lankuohsing https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html#sklearn.feature_extraction.text.TfidfVectorizer.fit_transform https://scikit-learn.org/stable/modules/feature_extraction.html#text-feature-extraction # In[] # In[] " sklearn implementation # L2 regularization y default # In[] #print(X) # sentence embeddings of corpus # In[] # In[] # In[] # In[] implementation by numpy and math # In[] # In[] # In[] # In[] # sentence Embeddings of corpus # In[] # should be all zeros