### PREPROCESSOR ###
# TODO COMMENTING
# TODO REPORTING AND ANALYSIS
# TODO PANDAS INTEGRATION
# TODO FIX DESCRIPTIONS
#? THREADING?
#? ERROR HANDLING?
#? DO UNIT TESTS?

#* LIBRARIES
from lib.preprocess_utils import *
from lib.options import O
import lib.pos_tagger as pos_tagger

from googletrans import Translator
from textblob import TextBlob
from tqdm import tqdm
from snowballstemmer import TurkishStemmer as sbts
import simplemma as sl
import pandas as pd
import numpy as np
import glob, json, mmap, threading, queue

#* ROUTINE
# This is created for using in different modules
def preprocess_routine(MODE:int, 
                       RESET:str, 
                       tm:tqdm = tqdm(total=2, unit='operation', desc="Reset"), 
                       t:tqdm = tqdm(unit='file'), 
                       tl:tqdm = tqdm(unit='line')) -> None:
    """The preprocess routine."""
    #* INITIALIZING

    #* RESET PROGRESS
    # Resets out dir, removing every file
    #! THIS TRAINS POS_TAGGER AGAIN
    if RESET == 'y':
        reset_out(t) # Reset out directory
        tm.update() # Update loading bar

        pos_tagger.init(t, tl) # Generate feature list POS_TRAINING_DATA for training CRF
        pos_model = pos_tagger.train_pos_model(t) # Train and save POS_CRF_MODEL
        pos_tagger.save_pos_model(pos_model) # Save POS_MODEL
        tm.update() # Update loading bar

        del pos_model # Delete POS_MODEL for more memory 

    tm.set_description("Preprocessing tweets")
    tm.reset(13)

    #* DATA SELECTION
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    # Update loading bar
    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Selecting data")

    for file in json_files:
        index_val = 0
        
        # Update loading bar
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    old_json_obj = json.loads(line) # Get json object from line
                    
                    if old_json_obj['language'] != 'tr': # Skip if tweet isn't Turkish
                        tl.update()
                        continue
                    json_obj = {}

                    # Load required fields
                    json_obj['id'] = index_val
                    json_obj['tweet'] = fixChars(old_json_obj['tweet'])
                    json_obj['replies'] = old_json_obj['replies_count']
                    json_obj['retweets'] = old_json_obj['retweets_count']
                    json_obj['likes'] = old_json_obj['likes_count']
                    
                    # Save new line
                    new_json_obj_str = json.dumps(json_obj, ensure_ascii=False)
                    newfile.write(new_json_obj_str + "\n")
                    index_val += 1
                    tl.update()
        t.update()

    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* TOKENIZATION
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    # Update loading bar
    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Tokenizing")

    for file in json_files:
        # Update loading bar
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    json_obj = json.loads(line) # Get json object from line

                    tokens = []
                    for token in json_obj['tweet'].split(" "): # Split sentence
                        if token not in ['', ' ']:
                            tokens.append(token)
                        
                    json_obj['tokens'] = tokens
                    
                    # Save new line
                    tokenized_json_obj = json.dumps(json_obj, ensure_ascii=False)
                    newfile.write(tokenized_json_obj + "\n")
                    tl.update()    
        t.update()

    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* NORMALIZATION
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    # Update loading bar
    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Normalizing")

    for file in json_files:
        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))

                for line in iter(mf.readline, b''):
                    json_obj = json.loads(line) # Get json object from line
                    new_tokens = []
                    hashtags = []
                    stopwords = []

                    for token in json_obj['tokens']:
                        # Normalize token
                        token = lowercaseTR(token)
                        token = deEmojify(token)
                        token = deNumerize(token)
                        token = deSoftChar(token)

                        if stopWordDetect(token): # Skip if token is a stop word
                            stopwords.append(token)
                            continue
                        elif '#' in token: # Skip if token is a hashtag
                            hashtags.append(token)
                            continue
                        elif 'http' in token: # Skip if token is a link
                            continue
                        elif '' == token or ' ' == token: # Skip if token is a newline or empty space
                            continue
                        elif '@' == token: # Skip if token is a mention
                            continue
                        elif '.' in token and '...' not in token: # There are probably two words here, split them.
                            mid_tokens = token.split('.')
                            for i in range(len(mid_tokens)):
                                mid_tokens[i] = dePunctuate(mid_tokens[i])
                            new_tokens.extend(mid_tokens)
                        else:
                            token = dePunctuate(token)
                            new_tokens.append(token)

                    if '' in new_tokens:
                        new_tokens.remove('') 
                    if ' ' in new_tokens:
                        new_tokens.remove(' ') 
                    json_obj['tokens'] = new_tokens
                    json_obj['hashtags'] = hashtags
                    json_obj['stop_words'] = stopwords
                    new_json_obj = json.dumps(json_obj, ensure_ascii=False)
                    newfile.write(new_json_obj + "\n")
                    tl.update()
        t.update()

    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* STEMMING
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")
    
    stemmer = sbts()

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Stemming")

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line
                    stemmed_tokens = []

                    for token in tweet['tokens']:
                        stemmed_token = stemmer.stemWord(token)
                        stemmed_tokens.append(stemmed_token)

                    tweet['stemmed_tokens'] = stemmed_tokens

                    new_json_obj = json.dumps(tweet, ensure_ascii=False)
                    newfile.write(new_json_obj + "\n")
                    tl.update()
        t.update()
        
    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* GENERATING STEM_DICT
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Generating STEM_DICT parts")

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))
        word_dict = {}

        with open(file, encoding="utf-8") as oldfile, open(O.STEM_DICT_PATH + "STEM_DICT_" + f_name(file), 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line

                    for i in range(len(tweet['stemmed_tokens'])):
                        if tweet['stemmed_tokens'][i] == '':
                            continue
                        if tweet['stemmed_tokens'][i] not in word_dict.keys():
                            word_dict[tweet['stemmed_tokens'][i]] = []
                        if tweet['tokens'][i] not in word_dict[tweet['stemmed_tokens'][i]]:
                            word_dict[tweet['stemmed_tokens'][i]].append(tweet['tokens'][i])
                    tl.update()

            newfile.write(json.dumps(word_dict, indent=2, ensure_ascii=False))
        t.update()

    stem_files = glob.glob(O.STEM_DICT_PATH + "*.json")

    t.reset(len(stem_files))
    t.unit = 'file'
    t.set_description("Combining STEM_DICT parts")

    stem_dict = {}
    for file in stem_files:
        with open(file, encoding="utf-8") as oldfile:
            raw_json:dict = json.load(oldfile)

            tl.reset(len(raw_json.keys()))
            tl.set_description("Processing {}".format(f_name(file)))

            for k, v in raw_json.items():
                tl.update()
                if k in stem_dict.keys():
                    stem_dict[k].extend(v)
                else:
                    stem_dict[k] = v
                tl.update()
        t.update()

    tl.reset(len(stem_dict))
    tl.set_description("Dropping duplicates")
    for k, v in stem_dict.items():
        stem_dict[k] = list(set(v))

    with open(O.STEM_DICT_PATH + "STEM_DICT.json", 'w', encoding="utf-8") as newfile:
        json.dump(stem_dict, newfile, ensure_ascii=False, indent=2)

    tm.update() # Update loading bar
    del stem_dict

    #* LEMMATIZING
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Lemmatizing")

    lang_data = sl.load_data('tr')

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line
                    lemmatized_tokens = []

                    for token in tweet['tokens']:
                        try:
                            lemmatized_token = sl.lemmatize(token, lang_data)
                        except:
                            lemmatized_token = ''
                        lemmatized_tokens.append(lemmatized_token)

                    tweet['lemmatized_tokens'] = lemmatized_tokens

                    new_json_obj = json.dumps(tweet, ensure_ascii=False)
                    newfile.write(new_json_obj + "\n")
                    tl.update()
        t.update()
        
    rename_NEW(tl)
    tm.update() # Update loading bar

    #* GENERATING LEMMA_DICT
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Generating LEMMA_DICT parts")

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))
        word_dict = {}

        with open(file, encoding="utf-8") as oldfile, open(O.LEMMA_DICT_PATH + "LEMMA_DICT_" + f_name(file), 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line

                    for i in range(len(tweet['lemmatized_tokens'])):
                        if tweet['lemmatized_tokens'][i] == '':
                            continue
                        if tweet['lemmatized_tokens'][i] not in word_dict.keys():
                            word_dict[tweet['lemmatized_tokens'][i]] = []
                        if tweet['tokens'][i] not in word_dict[tweet['lemmatized_tokens'][i]]:
                            word_dict[tweet['lemmatized_tokens'][i]].append(tweet['tokens'][i])
                    tl.update()

            newfile.write(json.dumps(word_dict, indent=2, ensure_ascii=False))
        t.update()

    lemma_files = glob.glob(O.LEMMA_DICT_PATH + "*.json")

    t.reset(len(lemma_files))
    t.unit = 'file'
    t.set_description("Combining LEMMA_DICT parts")

    lemma_dict = {}
    for file in lemma_files:
        with open(file, encoding="utf-8") as oldfile, open(O.LEMMA_DICT_PATH + "LEMMA_DICT.json", 'a', encoding="utf-8") as newfile:
            raw_json:dict = json.load(oldfile)

            tl.reset(len(raw_json.keys()))
            tl.set_description("Processing {}".format(f_name(file)))

            for k, v in raw_json.items():
                tl.update()
                if k in lemma_dict.keys():
                    lemma_dict[k].extend(v)
                else:
                    lemma_dict[k] = v
                tl.update()
        t.update()    

    tl.reset(len(lemma_dict))
    tl.set_description("Dropping duplicates")
    for k, v in lemma_dict.items():
        lemma_dict[k] = list(set(v))

    with open(O.LEMMA_DICT_PATH + "LEMMA_DICT.json", 'a', encoding="utf-8") as newfile:
        json.dump(lemma_dict, newfile, ensure_ascii=False, indent=2)

    tm.update() # Update loading bar
    del lemma_dict

    #* GENERATE WORD_DICT

    stem_dict_file = O.STEM_DICT_PATH + "STEM_DICT.json"
    lemma_dict_file = O.LEMMA_DICT_PATH + "LEMMA_DICT.json"

    t.reset(2)
    t.unit = 'file'
    t.set_description("Generating WORD_DICT")

    stem_file = open(stem_dict_file, encoding="utf-8")
    lemma_file = open(lemma_dict_file, encoding="utf-8")
    sfm = mmap.mmap(stem_file.fileno(), 0, access=mmap.ACCESS_READ)
    lfm = mmap.mmap(lemma_file.fileno(), 0, access=mmap.ACCESS_READ)
    stem_dict:dict = json.load(sfm)
    lemma_dict:dict = json.load(lfm)
    sfm.close()
    lfm.close()

    word_dict = {}
    for k, v in stem_dict.items():
        tl.reset(len(stem_dict.values()))
        tl.set_description("Processing STEM_DICT")
        for word in v:
            if word not in word_dict.keys():
                word_dict[word] = {}
            word_dict[word]['stem'] = k
        tl.update()

    t.update()

    for k, v in lemma_dict.items():
        tl.reset(len(lemma_dict.values()))
        tl.set_description("Processing LEMMA_DICT")
        for word in v:
            if word not in word_dict.keys():
                word_dict[word] = {'stem':'', 'lemma':''}
            word_dict[word]['lemma'] = k
        tl.update()

    t.update()

    with open(O.WORD_DICT_PATH, 'w', encoding="utf-8") as wdf:
        wdf.write(json.dumps(word_dict, indent=2, ensure_ascii=False))

    tm.update() # Update loading bar

    #* POS TAGGING   
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")
    
    pt = pos_tagger.load_pos_model() 

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("POS Tagging Tweets")

    for file in json_files:
        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))

                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line
                    sentence = []

                    for i in range(len(tweet['tokens'])):
                        sentence.append([i, tweet['tokens'][i], tweet['lemmatized_tokens'][i], ''])

                    for i in range(len(sentence)):
                        feature = pos_tagger.features(sentence, i)
                        tag = pt.predict([[feature]])[0][0]
                        sentence[i][3] = tag

                    tags = []
                    for token in sentence:
                        tags.append(token[3])

                    tweet['pos_tags'] = tags
                    newfile.write(json.dumps(tweet, ensure_ascii=False) + '\n')  
                    tl.update()
        t.update() 
    rename_NEW(t)

    tm.update() # Update loading bar

    #* POLARITY GENERATION
    if MODE == 1:
        def polarity_routine(tweet_batch:list[dict], file_path:str, tl:tqdm_notebook):
            translator = Translator()
            with open(file_path, 'a', encoding="utf-8") as newfile:
                for tweet in tweet_batch:
                    try:
                        combined_tokens = ' '.join(tweet['tokens'])
                        english_combined_tokens = translator.translate(combined_tokens, dest='en').text
                        tb = TextBlob(english_combined_tokens)
                        sentiment = tb.sentiment
                        tweet['polarity'] = 1.0 if sentiment.polarity > 3.4e-1 else -1.0 if sentiment.polarity < -2.73e-1 else 0.0
                        tweet['subjectivity'] = sentiment.subjectivity
                        newfile.write(json.dumps(tweet, ensure_ascii=False) + '\n')
                        tl.update()
                    except Exception:
                        tweet['polarity'] = 0.0
                        tweet['subjectivity'] = 0.0
                        newfile.write(json.dumps(tweet, ensure_ascii=False) + '\n')
                        tl.update()

        json_files = glob.glob(O.TWEETS_PATH+"*.json")

        t.reset(len(json_files))
        t.unit = 'file'
        t.set_description("Generating polarities")

        for file in json_files:
            tl.reset(count_lines(file))
            tl.set_description("Processing {}".format(f_name(file)))
            tweets = []     

            with open(file, encoding="utf-8") as oldfile:
                with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                    for line in iter(mf.readline, b''):
                        tweets.append(json.loads(line)) # Get json object from line

            tweet_batches = split(tweets)

            jobs = [threading.Thread(target=polarity_routine, args=(batch, file+f".NEW_{tweet_batches.index(batch)}", tl)) 
                    for batch in tweet_batches]

            for job in jobs:
                job.start()

            for job in jobs:  
                job.join() 
   
            del tweet_batches
            t.update()

        combine_NEW(t)
        rename_NEW(t)            
    tm.update() # Update loading bar  
    
    #* EXTRACTING TOKEN CORPUS
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Generating corpus")
    
    corpus = []
    
    for file in json_files:        
        with open(file, encoding="utf-8") as oldfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))
                
                for line in iter(mf.readline, b''):
                    obj = json.loads(line) # Get json object from line
                    corpus.append((" ".join(obj['tokens']), obj['polarity'], obj['subjectivity']))
                    
                tl.update()                    
        t.update()
        
    with open(O.CORPUS_PATH, 'w', encoding='utf-8') as corpusfile:
        for cp in corpus:
            corpusfile.write(cp[0] + "\n")
        
    tm.update() # Update loading bar    
    
    #* CALCULATING IDF-POLARITY AND IDF-SUBJECTIVITY
    t.reset(2)
    t.unit = 'file'
    t.set_description("Extracting word polarities")
    
    tl.reset(6)
    tl.set_description("Processing".format(f_name(file)))
                
    corpus = pd.DataFrame(corpus, columns=["Tweet", "Polarity", "Subjectivity"])
    tl.update()
    
    corpus_freq = corpus['Tweet'].str.split(expand=True).stack().value_counts().reset_index()
    corpus_freq.columns = ["Word", "Frequency"]
    corpus_freq = corpus_freq.set_index("Word")
    tl.update()
    
    corpus['Positive Polarity'] = np.where(corpus['Polarity'] == 1.0, 1, 0)
    corpus['Neutral Polarity'] = np.where(corpus['Polarity'] == 0.0, 1, 0)
    corpus['Negative Polarity'] = np.where(corpus['Polarity'] == -1.0, 1, 0)
    corpus['Positive Subjectivity'] = np.where(corpus['Subjectivity'] >= 0.5, 1, 0)
    corpus['Negative Subjectivity'] = np.where(corpus['Subjectivity'] < 0.5, 1, 0)
    corpus = corpus.drop(columns=['Polarity', 'Subjectivity'])

    corpus['Tweet'] = corpus['Tweet'].str.split(' ')
    corpus = corpus.explode('Tweet')
    corpus = corpus.groupby('Tweet', as_index=False)[[
        "Positive Polarity", 
        "Neutral Polarity", 
        "Negative Polarity", 
        "Positive Subjectivity", 
        "Negative Subjectivity"]].sum()
    corpus = corpus.rename(columns={'Tweet':'Word'})
    tl.update()
    
    corpus = corpus.merge(corpus_freq, left_on='Word', right_index=True)
    corpus = corpus.dropna()
    corpus = corpus.sort_values(by=['Frequency'], ascending=False)
    corpus = corpus.reset_index(drop=True)
    tl.update()
    del corpus_freq
    
    corpus_len = len(corpus)
    corpus['IDF'] = np.log10(corpus_len / (1 + corpus['Frequency']))
    tl.update()
    
    corpus = corpus.set_index("Word")
    corpus = corpus.to_dict("index")
    tl.update()
    t.update()
    
    tl.set_description("Updating WORD_DICT")
    tl.reset(len(word_dict))
    
    for k, v in word_dict.items():
        try:
            v['word_freq'] = corpus[k]['Frequency']
            v['idf'] = corpus[k]['IDF']
            v['pos_freq'] = corpus[k]['Positive Polarity']
            v['net_freq'] = corpus[k]['Neutral Polarity']
            v['neg_freq'] = corpus[k]['Negative Polarity']
            v['pos_subj_freq'] = corpus[k]['Positive Subjectivity']
            v['neg_subj_freq'] = corpus[k]['Negative Subjectivity']
            
            word_dict[k] = v
        except:
            pass
        tl.update()
    
    with open(O.WORD_DICT_PATH, 'w', encoding='utf-8') as wdf:
        json.dump(word_dict, wdf, indent=4, ensure_ascii=False)
    
    t.update()
    tm.update()
    
    #* UPDATING STEM AND LEMMA DICTS
    stem_dict = {}
    with open(O.STEM_DICT_PATH + 'STEM_DICT.json', 'r', encoding='utf-8') as wdf:
        with mmap.mmap(wdf.fileno(), 0, access=mmap.ACCESS_READ) as mf:
            stem_dict = json.load(mf)

    for k, v in stem_dict.items():
        total_word_freq = 0
        total_pos_freq = 0
        total_net_freq = 0
        total_neg_freq = 0
        total_pos_subj_freq = 0
        total_neg_subj_freq = 0
        for w in v:
            total_word_freq += word_dict[w]['word_freq']
            total_pos_freq += word_dict[w]['pos_freq']
            total_net_freq += word_dict[w]['net_freq']
            total_neg_freq += word_dict[w]['neg_freq']
            total_pos_subj_freq += word_dict[w]['pos_subj_freq']
            total_neg_subj_freq += word_dict[w]['neg_subj_freq']
        
        new_v = {
            'word_freq':total_word_freq,
            'pos_freq':total_pos_freq,
            'net_freq':total_net_freq,
            'neg_freq':total_neg_freq,
            'pos_subj_freq':total_pos_subj_freq,
            'neg_subj_freq':total_neg_subj_freq
        }
        stem_dict[k] = new_v
    
    with open(O.FINAL_STEM_PATH, 'w', encoding='utf-8') as wdf:
        json.dump(stem_dict, wdf, indent=4, ensure_ascii=False)
        
    del stem_dict    
    
    lemma_dict = {}
    with open(O.LEMMA_DICT_PATH + 'LEMMA_DICT.json', 'r', encoding='utf-8') as wdf:
        with mmap.mmap(wdf.fileno(), 0, access=mmap.ACCESS_READ) as mf:
            lemma_dict = json.load(mf)

    for k, v in lemma_dict.items():
        total_word_freq = 0
        total_pos_freq = 0
        total_net_freq = 0
        total_neg_freq = 0
        total_pos_subj_freq = 0
        total_neg_subj_freq = 0
        for w in v:
            total_word_freq += word_dict[w]['word_freq']
            total_pos_freq += word_dict[w]['pos_freq']
            total_net_freq += word_dict[w]['net_freq']
            total_neg_freq += word_dict[w]['neg_freq']
            total_pos_subj_freq += word_dict[w]['pos_subj_freq']
            total_neg_subj_freq += word_dict[w]['neg_subj_freq']
        
        new_v = {
            'word_freq':total_word_freq,
            'pos_freq':total_pos_freq,
            'net_freq':total_net_freq,
            'neg_freq':total_neg_freq,
            'pos_subj_freq':total_pos_subj_freq,
            'neg_subj_freq':total_neg_subj_freq
        }
        lemma_dict[k] = new_v
    
    with open(O.FINAL_LEMMA_PATH, 'w', encoding='utf-8') as wdf:
        json.dump(lemma_dict, wdf, indent=4, ensure_ascii=False)
    
    del lemma_dict, word_dict
    
    #* FORMATTING
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Formatting")

    for file in json_files:
        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))

                newfile.write("[")
                for line in iter(mf.readline, b''):
                    obj = json.loads(line) # Get json object from line
                    newfile.write(json.dumps(obj, ensure_ascii=False, indent=4) + ",\n")
                newfile.seek(0, os.SEEK_END)
                newfile.seek(newfile.tell() - 3, os.SEEK_SET)
                newfile.write("]")
                tl.update()
        t.update()
    rename_NEW(t)

    tm.update() # Update loading bar      
    
#* PROMPT
# Get preferences for preprocessing if called from console
if __name__ == '__main__':
    from os import system
    try:
        print("### PREPROCESSOR ###\n")
        print("1) Preprocess training tweets")
        print("2) Preprocess normal tweets\n")
        MODE = int(input("Select preprocessing mode: "))
        if MODE not in [1, 2]:
            raise Exception()

        print("Do you want to reset out directory?")
        print("Note that this will also delete trainer models.")
        RESET = input("Select [Y]es/[N]o: ").lower()
        if RESET not in ['y', 'n']:
            raise Exception()
        
    except Exception:
        print("Unknown choice. CLOSING...")
        exit(1)
    
    else:
        system("cls")
        preprocess_routine(MODE, RESET)
### PREPROCESSOR ###
# TODO COMMENTING
# TODO REPORTING AND ANALYSIS
# TODO PANDAS INTEGRATION
# TODO FIX DESCRIPTIONS
#? THREADING?
#? ERROR HANDLING?
#? DO UNIT TESTS?

#* LIBRARIES
from lib.preprocess_utils import *
from lib.options import O
import lib.pos_tagger as pos_tagger

from googletrans import Translator
from textblob import TextBlob
from tqdm import tqdm
from snowballstemmer import TurkishStemmer as sbts
import simplemma as sl
import pandas as pd
import numpy as np
import glob, json, mmap, threading, queue

#* ROUTINE
# This is created for using in different modules
def preprocess_routine(MODE:int, 
                       RESET:str, 
                       tm:tqdm = tqdm(total=2, unit='operation', desc="Reset"), 
                       t:tqdm = tqdm(unit='file'), 
                       tl:tqdm = tqdm(unit='line')) -> None:
    """The preprocess routine."""
    #* INITIALIZING

    #* RESET PROGRESS
    # Resets out dir, removing every file
    #! THIS TRAINS POS_TAGGER AGAIN
    if RESET == 'y':
        reset_out(t) # Reset out directory
        tm.update() # Update loading bar

        pos_tagger.init(t, tl) # Generate feature list POS_TRAINING_DATA for training CRF
        pos_model = pos_tagger.train_pos_model(t) # Train and save POS_CRF_MODEL
        pos_tagger.save_pos_model(pos_model) # Save POS_MODEL
        tm.update() # Update loading bar

        del pos_model # Delete POS_MODEL for more memory 

    tm.set_description("Preprocessing tweets")
    tm.reset(13)

    #* DATA SELECTION
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    # Update loading bar
    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Selecting data")

    for file in json_files:
        index_val = 0
        
        # Update loading bar
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    old_json_obj = json.loads(line) # Get json object from line
                    
                    if old_json_obj['language'] != 'tr': # Skip if tweet isn't Turkish
                        tl.update()
                        continue
                    json_obj = {}

                    # Load required fields
                    json_obj['id'] = index_val
                    json_obj['tweet'] = fixChars(old_json_obj['tweet'])
                    json_obj['replies'] = old_json_obj['replies_count']
                    json_obj['retweets'] = old_json_obj['retweets_count']
                    json_obj['likes'] = old_json_obj['likes_count']
                    
                    # Save new line
                    new_json_obj_str = json.dumps(json_obj, ensure_ascii=False)
                    newfile.write(new_json_obj_str + "\n")
                    index_val += 1
                    tl.update()
        t.update()

    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* TOKENIZATION
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    # Update loading bar
    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Tokenizing")

    for file in json_files:
        # Update loading bar
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    json_obj = json.loads(line) # Get json object from line

                    tokens = []
                    for token in json_obj['tweet'].split(" "): # Split sentence
                        if token not in ['', ' ']:
                            tokens.append(token)
                        
                    json_obj['tokens'] = tokens
                    
                    # Save new line
                    tokenized_json_obj = json.dumps(json_obj, ensure_ascii=False)
                    newfile.write(tokenized_json_obj + "\n")
                    tl.update()    
        t.update()

    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* NORMALIZATION
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    # Update loading bar
    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Normalizing")

    for file in json_files:
        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))

                for line in iter(mf.readline, b''):
                    json_obj = json.loads(line) # Get json object from line
                    new_tokens = []
                    hashtags = []
                    stopwords = []

                    for token in json_obj['tokens']:
                        # Normalize token
                        token = lowercaseTR(token)
                        token = deEmojify(token)
                        token = deNumerize(token)
                        token = deSoftChar(token)

                        if stopWordDetect(token): # Skip if token is a stop word
                            stopwords.append(token)
                            continue
                        elif '#' in token: # Skip if token is a hashtag
                            hashtags.append(token)
                            continue
                        elif 'http' in token: # Skip if token is a link
                            continue
                        elif '' == token or ' ' == token: # Skip if token is a newline or empty space
                            continue
                        elif '@' == token: # Skip if token is a mention
                            continue
                        elif '.' in token and '...' not in token: # There are probably two words here, split them.
                            mid_tokens = token.split('.')
                            for i in range(len(mid_tokens)):
                                mid_tokens[i] = dePunctuate(mid_tokens[i])
                            new_tokens.extend(mid_tokens)
                        else:
                            token = dePunctuate(token)
                            new_tokens.append(token)

                    if '' in new_tokens:
                        new_tokens.remove('') 
                    if ' ' in new_tokens:
                        new_tokens.remove(' ') 
                    json_obj['tokens'] = new_tokens
                    json_obj['hashtags'] = hashtags
                    json_obj['stop_words'] = stopwords
                    new_json_obj = json.dumps(json_obj, ensure_ascii=False)
                    newfile.write(new_json_obj + "\n")
                    tl.update()
        t.update()

    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* STEMMING
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")
    
    stemmer = sbts()

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Stemming")

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line
                    stemmed_tokens = []

                    for token in tweet['tokens']:
                        stemmed_token = stemmer.stemWord(token)
                        stemmed_tokens.append(stemmed_token)

                    tweet['stemmed_tokens'] = stemmed_tokens

                    new_json_obj = json.dumps(tweet, ensure_ascii=False)
                    newfile.write(new_json_obj + "\n")
                    tl.update()
        t.update()
        
    rename_NEW(tl) # Update tweet files
    tm.update() # Update loading bar

    #* GENERATING STEM_DICT
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Generating STEM_DICT parts")

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))
        word_dict = {}

        with open(file, encoding="utf-8") as oldfile, open(O.STEM_DICT_PATH + "STEM_DICT_" + f_name(file), 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line

                    for i in range(len(tweet['stemmed_tokens'])):
                        if tweet['stemmed_tokens'][i] == '':
                            continue
                        if tweet['stemmed_tokens'][i] not in word_dict.keys():
                            word_dict[tweet['stemmed_tokens'][i]] = []
                        if tweet['tokens'][i] not in word_dict[tweet['stemmed_tokens'][i]]:
                            word_dict[tweet['stemmed_tokens'][i]].append(tweet['tokens'][i])
                    tl.update()

            newfile.write(json.dumps(word_dict, indent=2, ensure_ascii=False))
        t.update()

    stem_files = glob.glob(O.STEM_DICT_PATH + "*.json")

    t.reset(len(stem_files))
    t.unit = 'file'
    t.set_description("Combining STEM_DICT parts")

    stem_dict = {}
    for file in stem_files:
        with open(file, encoding="utf-8") as oldfile:
            raw_json:dict = json.load(oldfile)

            tl.reset(len(raw_json.keys()))
            tl.set_description("Processing {}".format(f_name(file)))

            for k, v in raw_json.items():
                tl.update()
                if k in stem_dict.keys():
                    stem_dict[k].extend(v)
                else:
                    stem_dict[k] = v
                tl.update()
        t.update()

    tl.reset(len(stem_dict))
    tl.set_description("Dropping duplicates")
    for k, v in stem_dict.items():
        stem_dict[k] = list(set(v))

    with open(O.STEM_DICT_PATH + "STEM_DICT.json", 'w', encoding="utf-8") as newfile:
        json.dump(stem_dict, newfile, ensure_ascii=False, indent=2)

    tm.update() # Update loading bar
    del stem_dict

    #* LEMMATIZING
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Lemmatizing")

    lang_data = sl.load_data('tr')

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))

        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line
                    lemmatized_tokens = []

                    for token in tweet['tokens']:
                        try:
                            lemmatized_token = sl.lemmatize(token, lang_data)
                        except:
                            lemmatized_token = ''
                        lemmatized_tokens.append(lemmatized_token)

                    tweet['lemmatized_tokens'] = lemmatized_tokens

                    new_json_obj = json.dumps(tweet, ensure_ascii=False)
                    newfile.write(new_json_obj + "\n")
                    tl.update()
        t.update()
        
    rename_NEW(tl)
    tm.update() # Update loading bar

    #* GENERATING LEMMA_DICT
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Generating LEMMA_DICT parts")

    for file in json_files:
        tl.reset(count_lines(file))
        tl.set_description("Processing {}".format(f_name(file)))
        word_dict = {}

        with open(file, encoding="utf-8") as oldfile, open(O.LEMMA_DICT_PATH + "LEMMA_DICT_" + f_name(file), 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line

                    for i in range(len(tweet['lemmatized_tokens'])):
                        if tweet['lemmatized_tokens'][i] == '':
                            continue
                        if tweet['lemmatized_tokens'][i] not in word_dict.keys():
                            word_dict[tweet['lemmatized_tokens'][i]] = []
                        if tweet['tokens'][i] not in word_dict[tweet['lemmatized_tokens'][i]]:
                            word_dict[tweet['lemmatized_tokens'][i]].append(tweet['tokens'][i])
                    tl.update()

            newfile.write(json.dumps(word_dict, indent=2, ensure_ascii=False))
        t.update()

    lemma_files = glob.glob(O.LEMMA_DICT_PATH + "*.json")

    t.reset(len(lemma_files))
    t.unit = 'file'
    t.set_description("Combining LEMMA_DICT parts")

    lemma_dict = {}
    for file in lemma_files:
        with open(file, encoding="utf-8") as oldfile, open(O.LEMMA_DICT_PATH + "LEMMA_DICT.json", 'a', encoding="utf-8") as newfile:
            raw_json:dict = json.load(oldfile)

            tl.reset(len(raw_json.keys()))
            tl.set_description("Processing {}".format(f_name(file)))

            for k, v in raw_json.items():
                tl.update()
                if k in lemma_dict.keys():
                    lemma_dict[k].extend(v)
                else:
                    lemma_dict[k] = v
                tl.update()
        t.update()    

    tl.reset(len(lemma_dict))
    tl.set_description("Dropping duplicates")
    for k, v in lemma_dict.items():
        lemma_dict[k] = list(set(v))

    with open(O.LEMMA_DICT_PATH + "LEMMA_DICT.json", 'a', encoding="utf-8") as newfile:
        json.dump(lemma_dict, newfile, ensure_ascii=False, indent=2)

    tm.update() # Update loading bar
    del lemma_dict

    #* GENERATE WORD_DICT

    stem_dict_file = O.STEM_DICT_PATH + "STEM_DICT.json"
    lemma_dict_file = O.LEMMA_DICT_PATH + "LEMMA_DICT.json"

    t.reset(2)
    t.unit = 'file'
    t.set_description("Generating WORD_DICT")

    stem_file = open(stem_dict_file, encoding="utf-8")
    lemma_file = open(lemma_dict_file, encoding="utf-8")
    sfm = mmap.mmap(stem_file.fileno(), 0, access=mmap.ACCESS_READ)
    lfm = mmap.mmap(lemma_file.fileno(), 0, access=mmap.ACCESS_READ)
    stem_dict:dict = json.load(sfm)
    lemma_dict:dict = json.load(lfm)
    sfm.close()
    lfm.close()

    word_dict = {}
    for k, v in stem_dict.items():
        tl.reset(len(stem_dict.values()))
        tl.set_description("Processing STEM_DICT")
        for word in v:
            if word not in word_dict.keys():
                word_dict[word] = {}
            word_dict[word]['stem'] = k
        tl.update()

    t.update()

    for k, v in lemma_dict.items():
        tl.reset(len(lemma_dict.values()))
        tl.set_description("Processing LEMMA_DICT")
        for word in v:
            if word not in word_dict.keys():
                word_dict[word] = {'stem':'', 'lemma':''}
            word_dict[word]['lemma'] = k
        tl.update()

    t.update()

    with open(O.WORD_DICT_PATH, 'w', encoding="utf-8") as wdf:
        wdf.write(json.dumps(word_dict, indent=2, ensure_ascii=False))

    tm.update() # Update loading bar

    #* POS TAGGING   
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")
    
    pt = pos_tagger.load_pos_model() 

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("POS Tagging Tweets")

    for file in json_files:
        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))

                for line in iter(mf.readline, b''):
                    tweet = json.loads(line) # Get json object from line
                    sentence = []

                    for i in range(len(tweet['tokens'])):
                        sentence.append([i, tweet['tokens'][i], tweet['lemmatized_tokens'][i], ''])

                    for i in range(len(sentence)):
                        feature = pos_tagger.features(sentence, i)
                        tag = pt.predict([[feature]])[0][0]
                        sentence[i][3] = tag

                    tags = []
                    for token in sentence:
                        tags.append(token[3])

                    tweet['pos_tags'] = tags
                    newfile.write(json.dumps(tweet, ensure_ascii=False) + '\n')  
                    tl.update()
        t.update() 
    rename_NEW(t)

    tm.update() # Update loading bar

    #* POLARITY GENERATION
    if MODE == 1:
        def polarity_routine(tweet_batch:list[dict], file_path:str, tl:tqdm_notebook):
            translator = Translator()
            with open(file_path, 'a', encoding="utf-8") as newfile:
                for tweet in tweet_batch:
                    try:
                        combined_tokens = ' '.join(tweet['tokens'])
                        english_combined_tokens = translator.translate(combined_tokens, dest='en').text
                        tb = TextBlob(english_combined_tokens)
                        sentiment = tb.sentiment
                        tweet['polarity'] = 1.0 if sentiment.polarity > 3.4e-1 else -1.0 if sentiment.polarity < -2.73e-1 else 0.0
                        tweet['subjectivity'] = sentiment.subjectivity
                        newfile.write(json.dumps(tweet, ensure_ascii=False) + '\n')
                        tl.update()
                    except Exception:
                        tweet['polarity'] = 0.0
                        tweet['subjectivity'] = 0.0
                        newfile.write(json.dumps(tweet, ensure_ascii=False) + '\n')
                        tl.update()

        json_files = glob.glob(O.TWEETS_PATH+"*.json")

        t.reset(len(json_files))
        t.unit = 'file'
        t.set_description("Generating polarities")

        for file in json_files:
            tl.reset(count_lines(file))
            tl.set_description("Processing {}".format(f_name(file)))
            tweets = []     

            with open(file, encoding="utf-8") as oldfile:
                with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                    for line in iter(mf.readline, b''):
                        tweets.append(json.loads(line)) # Get json object from line

            tweet_batches = split(tweets)

            jobs = [threading.Thread(target=polarity_routine, args=(batch, file+f".NEW_{tweet_batches.index(batch)}", tl)) 
                    for batch in tweet_batches]

            for job in jobs:
                job.start()

            for job in jobs:  
                job.join() 
   
            del tweet_batches
            t.update()

        combine_NEW(t)
        rename_NEW(t)            
    tm.update() # Update loading bar  
    
    #* EXTRACTING TOKEN CORPUS
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Generating corpus")
    
    corpus = []
    
    for file in json_files:        
        with open(file, encoding="utf-8") as oldfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))
                
                for line in iter(mf.readline, b''):
                    obj = json.loads(line) # Get json object from line
                    corpus.append((" ".join(obj['tokens']), obj['polarity'], obj['subjectivity']))
                    
                tl.update()                    
        t.update()
        
    with open(O.CORPUS_PATH, 'w', encoding='utf-8') as corpusfile:
        for cp in corpus:
            corpusfile.write(cp[0] + "\n")
        
    tm.update() # Update loading bar    
    
    #* CALCULATING IDF-POLARITY AND IDF-SUBJECTIVITY
    t.reset(2)
    t.unit = 'file'
    t.set_description("Extracting word polarities")
    
    tl.reset(6)
    tl.set_description("Processing".format(f_name(file)))
                
    corpus = pd.DataFrame(corpus, columns=["Tweet", "Polarity", "Subjectivity"])
    tl.update()
    
    corpus_freq = corpus['Tweet'].str.split(expand=True).stack().value_counts().reset_index()
    corpus_freq.columns = ["Word", "Frequency"]
    corpus_freq = corpus_freq.set_index("Word")
    tl.update()
    
    corpus['Positive Polarity'] = np.where(corpus['Polarity'] == 1.0, 1, 0)
    corpus['Neutral Polarity'] = np.where(corpus['Polarity'] == 0.0, 1, 0)
    corpus['Negative Polarity'] = np.where(corpus['Polarity'] == -1.0, 1, 0)
    corpus['Positive Subjectivity'] = np.where(corpus['Subjectivity'] >= 0.5, 1, 0)
    corpus['Negative Subjectivity'] = np.where(corpus['Subjectivity'] < 0.5, 1, 0)
    corpus = corpus.drop(columns=['Polarity', 'Subjectivity'])

    corpus['Tweet'] = corpus['Tweet'].str.split(' ')
    corpus = corpus.explode('Tweet')
    corpus = corpus.groupby('Tweet', as_index=False)[[
        "Positive Polarity", 
        "Neutral Polarity", 
        "Negative Polarity", 
        "Positive Subjectivity", 
        "Negative Subjectivity"]].sum()
    corpus = corpus.rename(columns={'Tweet':'Word'})
    tl.update()
    
    corpus = corpus.merge(corpus_freq, left_on='Word', right_index=True)
    corpus = corpus.dropna()
    corpus = corpus.sort_values(by=['Frequency'], ascending=False)
    corpus = corpus.reset_index(drop=True)
    tl.update()
    del corpus_freq
    
    corpus_len = len(corpus)
    corpus['IDF'] = np.log10(corpus_len / (1 + corpus['Frequency']))
    tl.update()
    
    corpus = corpus.set_index("Word")
    corpus = corpus.to_dict("index")
    tl.update()
    t.update()
    
    tl.set_description("Updating WORD_DICT")
    tl.reset(len(word_dict))
    
    for k, v in word_dict.items():
        try:
            v['word_freq'] = corpus[k]['Frequency']
            v['idf'] = corpus[k]['IDF']
            v['pos_freq'] = corpus[k]['Positive Polarity']
            v['net_freq'] = corpus[k]['Neutral Polarity']
            v['neg_freq'] = corpus[k]['Negative Polarity']
            v['pos_subj_freq'] = corpus[k]['Positive Subjectivity']
            v['neg_subj_freq'] = corpus[k]['Negative Subjectivity']
            
            word_dict[k] = v
        except:
            pass
        tl.update()
    
    with open(O.WORD_DICT_PATH, 'w', encoding='utf-8') as wdf:
        json.dump(word_dict, wdf, indent=4, ensure_ascii=False)
    
    t.update()
    tm.update()
    
    #* UPDATING STEM AND LEMMA DICTS
    stem_dict = {}
    with open(O.STEM_DICT_PATH + 'STEM_DICT.json', 'r', encoding='utf-8') as wdf:
        with mmap.mmap(wdf.fileno(), 0, access=mmap.ACCESS_READ) as mf:
            stem_dict = json.load(mf)

    for k, v in stem_dict.items():
        total_word_freq = 0
        total_pos_freq = 0
        total_net_freq = 0
        total_neg_freq = 0
        total_pos_subj_freq = 0
        total_neg_subj_freq = 0
        for w in v:
            total_word_freq += word_dict[w]['word_freq']
            total_pos_freq += word_dict[w]['pos_freq']
            total_net_freq += word_dict[w]['net_freq']
            total_neg_freq += word_dict[w]['neg_freq']
            total_pos_subj_freq += word_dict[w]['pos_subj_freq']
            total_neg_subj_freq += word_dict[w]['neg_subj_freq']
        
        new_v = {
            'word_freq':total_word_freq,
            'pos_freq':total_pos_freq,
            'net_freq':total_net_freq,
            'neg_freq':total_neg_freq,
            'pos_subj_freq':total_pos_subj_freq,
            'neg_subj_freq':total_neg_subj_freq
        }
        stem_dict[k] = new_v
    
    with open(O.FINAL_STEM_PATH, 'w', encoding='utf-8') as wdf:
        json.dump(stem_dict, wdf, indent=4, ensure_ascii=False)
        
    del stem_dict    
    
    lemma_dict = {}
    with open(O.LEMMA_DICT_PATH + 'LEMMA_DICT.json', 'r', encoding='utf-8') as wdf:
        with mmap.mmap(wdf.fileno(), 0, access=mmap.ACCESS_READ) as mf:
            lemma_dict = json.load(mf)

    for k, v in lemma_dict.items():
        total_word_freq = 0
        total_pos_freq = 0
        total_net_freq = 0
        total_neg_freq = 0
        total_pos_subj_freq = 0
        total_neg_subj_freq = 0
        for w in v:
            total_word_freq += word_dict[w]['word_freq']
            total_pos_freq += word_dict[w]['pos_freq']
            total_net_freq += word_dict[w]['net_freq']
            total_neg_freq += word_dict[w]['neg_freq']
            total_pos_subj_freq += word_dict[w]['pos_subj_freq']
            total_neg_subj_freq += word_dict[w]['neg_subj_freq']
        
        new_v = {
            'word_freq':total_word_freq,
            'pos_freq':total_pos_freq,
            'net_freq':total_net_freq,
            'neg_freq':total_neg_freq,
            'pos_subj_freq':total_pos_subj_freq,
            'neg_subj_freq':total_neg_subj_freq
        }
        lemma_dict[k] = new_v
    
    with open(O.FINAL_LEMMA_PATH, 'w', encoding='utf-8') as wdf:
        json.dump(lemma_dict, wdf, indent=4, ensure_ascii=False)
    
    del lemma_dict, word_dict
    
    #* FORMATTING
    # Get tweets from TWEETS_PATH
    json_files = glob.glob(O.TWEETS_PATH + "*.json")

    t.reset(len(json_files))
    t.unit = 'file'
    t.set_description("Formatting")

    for file in json_files:
        with open(file, encoding="utf-8") as oldfile, open(file+".NEW", 'w', encoding="utf-8") as newfile:
            with mmap.mmap(oldfile.fileno(), 0, access=mmap.ACCESS_READ) as mf:
                tl.reset(count_lines(file))
                tl.set_description("Processing {}".format(f_name(file)))

                newfile.write("[")
                for line in iter(mf.readline, b''):
                    obj = json.loads(line) # Get json object from line
                    newfile.write(json.dumps(obj, ensure_ascii=False, indent=4) + ",\n")
                newfile.seek(0, os.SEEK_END)
                newfile.seek(newfile.tell() - 3, os.SEEK_SET)
                newfile.write("]")
                tl.update()
        t.update()
    rename_NEW(t)

    tm.update() # Update loading bar      
    
#* PROMPT
# Get preferences for preprocessing if called from console
if __name__ == '__main__':
    from os import system
    try:
        print("### PREPROCESSOR ###\n")
        print("1) Preprocess training tweets")
        print("2) Preprocess normal tweets\n")
        MODE = int(input("Select preprocessing mode: "))
        if MODE not in [1, 2]:
            raise Exception()

        print("Do you want to reset out directory?")
        print("Note that this will also delete trainer models.")
        RESET = input("Select [Y]es/[N]o: ").lower()
        if RESET not in ['y', 'n']:
            raise Exception()
        
    except Exception:
        print("Unknown choice. CLOSING...")
        exit(1)
    
    else:
        system("cls")
        preprocess_routine(MODE, RESET)
### PREPROCESSOR ### # TODO COMMENTING # TODO REPORTING AND ANALYSIS # TODO PANDAS INTEGRATION # TODO FIX DESCRIPTIONS #? THREADING? #? ERROR HANDLING? #? DO UNIT TESTS? #* LIBRARIES #* ROUTINE # This is created for using in different modules The preprocess routine. #* INITIALIZING #* RESET PROGRESS # Resets out dir, removing every file #! THIS TRAINS POS_TAGGER AGAIN # Reset out directory # Update loading bar # Generate feature list POS_TRAINING_DATA for training CRF # Train and save POS_CRF_MODEL # Save POS_MODEL # Update loading bar # Delete POS_MODEL for more memory #* DATA SELECTION # Get tweets from TWEETS_PATH # Update loading bar # Update loading bar # Get json object from line # Skip if tweet isn't Turkish # Load required fields # Save new line # Update tweet files # Update loading bar #* TOKENIZATION # Get tweets from TWEETS_PATH # Update loading bar # Update loading bar # Get json object from line # Split sentence # Save new line # Update tweet files # Update loading bar #* NORMALIZATION # Get tweets from TWEETS_PATH # Update loading bar # Get json object from line # Normalize token # Skip if token is a stop word # Skip if token is a hashtag # Skip if token is a link # Skip if token is a newline or empty space # Skip if token is a mention # There are probably two words here, split them. # Update tweet files # Update loading bar #* STEMMING # Get tweets from TWEETS_PATH # Get json object from line # Update tweet files # Update loading bar #* GENERATING STEM_DICT # Get tweets from TWEETS_PATH # Get json object from line # Update loading bar #* LEMMATIZING # Get tweets from TWEETS_PATH # Get json object from line # Update loading bar #* GENERATING LEMMA_DICT # Get tweets from TWEETS_PATH # Get json object from line # Update loading bar #* GENERATE WORD_DICT # Update loading bar #* POS TAGGING # Get tweets from TWEETS_PATH # Get json object from line # Update loading bar #* POLARITY GENERATION # Get json object from line # Update loading bar #* EXTRACTING TOKEN CORPUS # Get json object from line # Update loading bar #* CALCULATING IDF-POLARITY AND IDF-SUBJECTIVITY #* UPDATING STEM AND LEMMA DICTS #* FORMATTING # Get tweets from TWEETS_PATH # Get json object from line # Update loading bar #* PROMPT # Get preferences for preprocessing if called from console ## PREPROCESSOR ###\n")