<reponame>syazanihussin/FLUX
from development.A_DataPreparation import DataPreparation
from development.A_FeatureEngineering import FeatureEngineering
from development.A_Models import Models
from keras.preprocessing.text import text_to_word_sequence
from sklearn.feature_selection import chi2
from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt
import numpy as np
import malaya, csv, pandas
from system import search



''' COUNT TOTAL VOCAB & AVERAGE LENGTH '''

data_preparation = DataPreparation()
feature_engineering = FeatureEngineering()
model = Models()


scraped_content_data, train_x, test_x, train_y, test_y = data_preparation.prepare_data_normal_split(0.25)
scraped_stance_data, train_penyataan, test_penyataan, train_sumber, test_sumber, train_y2, test_y2 = data_preparation.prepare_stance_data_normal_split(0.25)

print(scraped_content_data)
print(scraped_stance_data)
print(len(scraped_content_data), len(scraped_stance_data))


scraped_content_list, scraped_stance_list = [], []
total_fake, total_real, total_news_avg = 0, 0, 0
total_fake2, total_real2, total_news_avg2 = 0, 0, 0


# VOCAB BEFORE STEMMING #
for i in range(len(scraped_content_data)):
    tokens = text_to_word_sequence(scraped_content_data[i], filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_content_list.append(token)

    if i%2==0:
        total_fake += len(scraped_content_data[i].split())
    elif i%2!=0:
        total_real += len(scraped_content_data[i].split())

    total_news_avg += total_fake + total_real


for i in range(len(scraped_stance_data)):
    tokens = text_to_word_sequence(scraped_stance_data[i], filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_stance_list.append(token)

    if i%2==0:
        total_fake2 += len(scraped_stance_data[i].split())
    elif i%2!=0:
        total_real2 += len(scraped_stance_data[i].split())

    total_news_avg2 += total_fake2 + total_real2


print(len(set(scraped_content_list)), set(scraped_content_list))
print(total_fake, total_real, total_news_avg)
print('avg fake news: ', total_fake/(len(scraped_content_data)/2))
print('avg real news: ', total_real/(len(scraped_content_data)/2))
print('avg total: ', total_news_avg/len(scraped_content_data))

print(len(set(scraped_stance_list)), set(scraped_stance_list))
print(total_fake2, total_real2, total_news_avg2)
print('avg fake news: ', total_fake2/(len(scraped_stance_data)/2))
print('avg real news: ', total_real2/(len(scraped_stance_data)/2))
print('avg total: ', total_news_avg2/len(scraped_stance_data))


scraped_content_list, scraped_stance_list = [], []
total_fake, total_real, total_news_avg = 0, 0, 0
total_fake2, total_real2, total_news_avg2 = 0, 0, 0


# VOCAB AFTER STEMMING #
for i in range(len(scraped_content_data)):
    news = malaya.sastrawi_stemmer(scraped_content_data[i])
    tokens = text_to_word_sequence(news, filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_content_list.append(token)

    if i%2==0:
        total_fake += len(news.split())
    elif i%2!=0:
        total_real += len(news.split())

    total_news_avg += total_fake + total_real


for i in range(len(scraped_stance_data)):
    news = malaya.sastrawi_stemmer(scraped_stance_data[i])
    tokens = text_to_word_sequence(news, filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_stance_list.append(token)

    if i%2==0:
        total_fake2 += len(news.split())
    elif i%2!=0:
        total_real2 += len(news.split())

    total_news_avg2 += total_fake2 + total_real2


print(len(set(scraped_content_list)), set(scraped_content_list))
print(total_fake, total_real, total_news_avg)
print('avg fake news: ', total_fake/(len(scraped_content_data)/2))
print('avg real news: ', total_real/(len(scraped_content_data)/2))
print('avg total: ', total_news_avg/len(scraped_content_data))

print(len(set(scraped_stance_list)), set(scraped_stance_list))
print(total_fake2, total_real2, total_news_avg2)
print('avg fake news: ', total_fake2/(len(scraped_stance_data)/2))
print('avg real news: ', total_real2/(len(scraped_stance_data)/2))
print('avg total: ', total_news_avg2/len(scraped_stance_data))



''' CREATE WORD VOCAB '''

def word_count(str):
    counts = dict()
    words = str.split()

    for word in words:
        if word in counts:
            counts[word] += 1
        else:
            counts[word] = 1

    return counts



''' TEST DIFFERENT EMBEDDING PERFORMANCE ON DIFFERENT DATASET '''

embedded = malaya.malaya_word2vec(256)

print(len(embedded['dictionary']), embedded['nce_weights'].shape)
word_vector = malaya.Word2Vec(embedded['nce_weights'], embedded['dictionary'])



''' FEATURE SELECTION '''

tvec = TfidfVectorizer(max_features=100000, ngram_range=(1, 3))
x_train_tfidf = tvec.fit_transform(train_x)
chi2score = chi2(x_train_tfidf, train_y)[0]

plt.figure(figsize=(15,10))
wscores = zip(tvec.get_feature_names(), chi2score)
wchi2 = sorted(wscores, key=lambda x:x[1])
topchi2 = list(zip(*wchi2[-20:]))
x = range(len(topchi2[1]))
labels = topchi2[0]
plt.barh(x,topchi2[1], align='center', alpha=0.2)
plt.plot(topchi2[1], x, '-o', markersize=5, alpha=0.8)
plt.yticks(x, labels)
plt.xlabel('$\chi^2$')


tvec = TfidfVectorizer(max_features=100000, ngram_range=(1, 3))
x_train_pen = tvec.fit_transform(train_penyataan)
x_train_sum = tvec.fit_transform(train_sumber)

x_train_pen = x_train_pen.toarray()
x_train_sum = x_train_sum.toarray()
concatenated = []
for data in range(len(x_train_sum)):
    concatenated.append(np.concatenate((x_train_pen, x_train_sum), axis=1))

chi2score = chi2(concatenated, train_y2)[0]

plt.figure(figsize=(15,10))
wscores = zip(tvec.get_feature_names(), chi2score)
wchi2 = sorted(wscores, key=lambda x:x[1])
topchi2 = list(zip(*wchi2[-20:]))
x = range(len(topchi2[1]))
labels = topchi2[0]
plt.barh(x,topchi2[1], align='center', alpha=0.2)
plt.plot(topchi2[1], x, '-o', markersize=5, alpha=0.8)
plt.yticks(x, labels)
plt.xlabel('$\chi^2$')
plt.show()



''' LDA TOPIC MODELLING '''

corpus = scraped_content_data.tolist()
lda2vec = malaya.lda2vec_topic_modelling(corpus, 10)
lda2vec.print_topics(5)
lda2vec.get_sentences(5)
lda2vec.visualize_topics(notebook_mode = True)



''' WRITING DATA TO CSV '''

with open('data500_csv.csv', mode='w', encoding='utf-8') as file:
    file_writer = csv.writer(file, delimiter=',', lineterminator = '\n')

    for i in range(len(scraped_content_data)):
        file_writer.writerow([scraped_content_data['fake_news'][i], 'Fake'])
        file_writer.writerow([scraped_content_data['real_news'][i], 'Real'])



''' COUNT LABEL DISTRIBUTION '''

content_data = pandas.read_csv('../data/content_csv.csv')
stance_data = pandas.read_csv('../data/stance_csv.csv')


print('RAW DATA')
print(content_data[['penyataan', 'label']])
print(stance_data[['penyataan','sumber', 'label']])

count_palsu_s, count_benar_s, count_xberkaitan_s, count_berkaitan_s = 0,0,0,0


for i in range(len(stance_data)):
    print(i)
    if(stance_data['label'].loc[i] == 'tidak setuju'):
        count_palsu_s+=1
    elif(stance_data['label'].loc[i] == 'setuju'):
        count_benar_s+=1
    elif(stance_data['label'].loc[i] == 'tidak berkaitan'):
        count_xberkaitan_s+=1
    elif(stance_data['label'].loc[i] == 'berkaitan'):
        count_berkaitan_s+=1


print(count_benar_s, count_palsu_s,count_berkaitan_s, count_xberkaitan_s)



''' SEARCH RELATED NEWS '''

searching_obj = search.Searching()

with open('../data/sumber_csv.csv', mode='w', encoding='utf-8') as file:
    file_writer = csv.writer(file, delimiter=',', lineterminator = '\n')

    for i in range(len(scraped_content_data)):
        searched_results = searching_obj.search_news(keyword=scraped_content_data[i])

        for searched_data in searched_results:
            print(searched_data['link'])
            print(searched_data['title'])
            print(searched_data['snippet'])
            print()
            key = i*2+1
            file_writer.writerow([key, searched_data['title'], searched_data['snippet'], searched_data['link']])
from development.A_DataPreparation import DataPreparation
from development.A_FeatureEngineering import FeatureEngineering
from development.A_Models import Models
from keras.preprocessing.text import text_to_word_sequence
from sklearn.feature_selection import chi2
from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt
import numpy as np
import malaya, csv, pandas
from system import search



''' COUNT TOTAL VOCAB & AVERAGE LENGTH '''

data_preparation = DataPreparation()
feature_engineering = FeatureEngineering()
model = Models()


scraped_content_data, train_x, test_x, train_y, test_y = data_preparation.prepare_data_normal_split(0.25)
scraped_stance_data, train_penyataan, test_penyataan, train_sumber, test_sumber, train_y2, test_y2 = data_preparation.prepare_stance_data_normal_split(0.25)

print(scraped_content_data)
print(scraped_stance_data)
print(len(scraped_content_data), len(scraped_stance_data))


scraped_content_list, scraped_stance_list = [], []
total_fake, total_real, total_news_avg = 0, 0, 0
total_fake2, total_real2, total_news_avg2 = 0, 0, 0


# VOCAB BEFORE STEMMING #
for i in range(len(scraped_content_data)):
    tokens = text_to_word_sequence(scraped_content_data[i], filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_content_list.append(token)

    if i%2==0:
        total_fake += len(scraped_content_data[i].split())
    elif i%2!=0:
        total_real += len(scraped_content_data[i].split())

    total_news_avg += total_fake + total_real


for i in range(len(scraped_stance_data)):
    tokens = text_to_word_sequence(scraped_stance_data[i], filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_stance_list.append(token)

    if i%2==0:
        total_fake2 += len(scraped_stance_data[i].split())
    elif i%2!=0:
        total_real2 += len(scraped_stance_data[i].split())

    total_news_avg2 += total_fake2 + total_real2


print(len(set(scraped_content_list)), set(scraped_content_list))
print(total_fake, total_real, total_news_avg)
print('avg fake news: ', total_fake/(len(scraped_content_data)/2))
print('avg real news: ', total_real/(len(scraped_content_data)/2))
print('avg total: ', total_news_avg/len(scraped_content_data))

print(len(set(scraped_stance_list)), set(scraped_stance_list))
print(total_fake2, total_real2, total_news_avg2)
print('avg fake news: ', total_fake2/(len(scraped_stance_data)/2))
print('avg real news: ', total_real2/(len(scraped_stance_data)/2))
print('avg total: ', total_news_avg2/len(scraped_stance_data))


scraped_content_list, scraped_stance_list = [], []
total_fake, total_real, total_news_avg = 0, 0, 0
total_fake2, total_real2, total_news_avg2 = 0, 0, 0


# VOCAB AFTER STEMMING #
for i in range(len(scraped_content_data)):
    news = malaya.sastrawi_stemmer(scraped_content_data[i])
    tokens = text_to_word_sequence(news, filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_content_list.append(token)

    if i%2==0:
        total_fake += len(news.split())
    elif i%2!=0:
        total_real += len(news.split())

    total_news_avg += total_fake + total_real


for i in range(len(scraped_stance_data)):
    news = malaya.sastrawi_stemmer(scraped_stance_data[i])
    tokens = text_to_word_sequence(news, filters='!"#$%&()*+,-./:;<=>?@[\]“”^_`{|}~‘’''\t\n\xa0â\x80\x9c\x9d\'')

    for token in tokens:
        scraped_stance_list.append(token)

    if i%2==0:
        total_fake2 += len(news.split())
    elif i%2!=0:
        total_real2 += len(news.split())

    total_news_avg2 += total_fake2 + total_real2


print(len(set(scraped_content_list)), set(scraped_content_list))
print(total_fake, total_real, total_news_avg)
print('avg fake news: ', total_fake/(len(scraped_content_data)/2))
print('avg real news: ', total_real/(len(scraped_content_data)/2))
print('avg total: ', total_news_avg/len(scraped_content_data))

print(len(set(scraped_stance_list)), set(scraped_stance_list))
print(total_fake2, total_real2, total_news_avg2)
print('avg fake news: ', total_fake2/(len(scraped_stance_data)/2))
print('avg real news: ', total_real2/(len(scraped_stance_data)/2))
print('avg total: ', total_news_avg2/len(scraped_stance_data))



''' CREATE WORD VOCAB '''

def word_count(str):
    counts = dict()
    words = str.split()

    for word in words:
        if word in counts:
            counts[word] += 1
        else:
            counts[word] = 1

    return counts



''' TEST DIFFERENT EMBEDDING PERFORMANCE ON DIFFERENT DATASET '''

embedded = malaya.malaya_word2vec(256)

print(len(embedded['dictionary']), embedded['nce_weights'].shape)
word_vector = malaya.Word2Vec(embedded['nce_weights'], embedded['dictionary'])



''' FEATURE SELECTION '''

tvec = TfidfVectorizer(max_features=100000, ngram_range=(1, 3))
x_train_tfidf = tvec.fit_transform(train_x)
chi2score = chi2(x_train_tfidf, train_y)[0]

plt.figure(figsize=(15,10))
wscores = zip(tvec.get_feature_names(), chi2score)
wchi2 = sorted(wscores, key=lambda x:x[1])
topchi2 = list(zip(*wchi2[-20:]))
x = range(len(topchi2[1]))
labels = topchi2[0]
plt.barh(x,topchi2[1], align='center', alpha=0.2)
plt.plot(topchi2[1], x, '-o', markersize=5, alpha=0.8)
plt.yticks(x, labels)
plt.xlabel('$\chi^2$')


tvec = TfidfVectorizer(max_features=100000, ngram_range=(1, 3))
x_train_pen = tvec.fit_transform(train_penyataan)
x_train_sum = tvec.fit_transform(train_sumber)

x_train_pen = x_train_pen.toarray()
x_train_sum = x_train_sum.toarray()
concatenated = []
for data in range(len(x_train_sum)):
    concatenated.append(np.concatenate((x_train_pen, x_train_sum), axis=1))

chi2score = chi2(concatenated, train_y2)[0]

plt.figure(figsize=(15,10))
wscores = zip(tvec.get_feature_names(), chi2score)
wchi2 = sorted(wscores, key=lambda x:x[1])
topchi2 = list(zip(*wchi2[-20:]))
x = range(len(topchi2[1]))
labels = topchi2[0]
plt.barh(x,topchi2[1], align='center', alpha=0.2)
plt.plot(topchi2[1], x, '-o', markersize=5, alpha=0.8)
plt.yticks(x, labels)
plt.xlabel('$\chi^2$')
plt.show()



''' LDA TOPIC MODELLING '''

corpus = scraped_content_data.tolist()
lda2vec = malaya.lda2vec_topic_modelling(corpus, 10)
lda2vec.print_topics(5)
lda2vec.get_sentences(5)
lda2vec.visualize_topics(notebook_mode = True)



''' WRITING DATA TO CSV '''

with open('data500_csv.csv', mode='w', encoding='utf-8') as file:
    file_writer = csv.writer(file, delimiter=',', lineterminator = '\n')

    for i in range(len(scraped_content_data)):
        file_writer.writerow([scraped_content_data['fake_news'][i], 'Fake'])
        file_writer.writerow([scraped_content_data['real_news'][i], 'Real'])



''' COUNT LABEL DISTRIBUTION '''

content_data = pandas.read_csv('../data/content_csv.csv')
stance_data = pandas.read_csv('../data/stance_csv.csv')


print('RAW DATA')
print(content_data[['penyataan', 'label']])
print(stance_data[['penyataan','sumber', 'label']])

count_palsu_s, count_benar_s, count_xberkaitan_s, count_berkaitan_s = 0,0,0,0


for i in range(len(stance_data)):
    print(i)
    if(stance_data['label'].loc[i] == 'tidak setuju'):
        count_palsu_s+=1
    elif(stance_data['label'].loc[i] == 'setuju'):
        count_benar_s+=1
    elif(stance_data['label'].loc[i] == 'tidak berkaitan'):
        count_xberkaitan_s+=1
    elif(stance_data['label'].loc[i] == 'berkaitan'):
        count_berkaitan_s+=1


print(count_benar_s, count_palsu_s,count_berkaitan_s, count_xberkaitan_s)



''' SEARCH RELATED NEWS '''

searching_obj = search.Searching()

with open('../data/sumber_csv.csv', mode='w', encoding='utf-8') as file:
    file_writer = csv.writer(file, delimiter=',', lineterminator = '\n')

    for i in range(len(scraped_content_data)):
        searched_results = searching_obj.search_news(keyword=scraped_content_data[i])

        for searched_data in searched_results:
            print(searched_data['link'])
            print(searched_data['title'])
            print(searched_data['snippet'])
            print()
            key = i*2+1
            file_writer.writerow([key, searched_data['title'], searched_data['snippet'], searched_data['link']])
COUNT TOTAL VOCAB & AVERAGE LENGTH # VOCAB BEFORE STEMMING # # VOCAB AFTER STEMMING # CREATE WORD VOCAB TEST DIFFERENT EMBEDDING PERFORMANCE ON DIFFERENT DATASET FEATURE SELECTION LDA TOPIC MODELLING WRITING DATA TO CSV COUNT LABEL DISTRIBUTION SEARCH RELATED NEWS