<filename>data_cleaner.py
#<NAME>
#2019
#Lit_Guru
#Data Cleaner/Transformer
#Will reduce dimensionality in the next script
import regex as re
import MySQLdb as mdb
import os
import enchant
dictionary=enchant.Dict('en_US')
story_list=os.listdir("../Stories/")
#loop through list here
for story in story_list:
    if(story!=".DS_Store"):
        print("\n"+story,end=' ')
        url="../Stories/"+story+"/"
        con = mdb.connect('localhost', 'root', 'password', '<PASSWORD>')
        with con.cursor() as cur:
            table_check=True
            #create tables here
            #each story will get its own table
            #table will consist of each word and their scores
            story_table_name=str(story)
            story_table_name=str(re.sub("[\-'\(\)\[\]}{:;!@\*\?></\$\+\|#_,\.]",'',story_table_name))
            #story names need to be escaped with back-quotes. Thanks, Change.
            try: cur.execute('CREATE TABLE `'+story_table_name+'` (word varchar(128),score double)')
            except mdb.Error as e:
                print("Error while creating table: "+story_table_name)
                table_check=False
            if(table_check):
                story_abstract={}
                #^will contain count of each valid word
                word_count=0
                #^will count total number of words
                j=1
                while(True):
                    print(j,end=' ')
                    try: f=open(url+str(j)+".txt")
                    except FileNotFoundError:
                        break
                    chapter=f.read()
                    #removes all non alphanumerics/spaces
                    chapter=str(re.sub('[^a-zA-Z\d\s]','',chapter))
                    word_bank=chapter.split()
                    for word in word_bank:
                        token=word.lower()
                        #puts word in lowercase and then checks that it's a valid english word
                        if(dictionary.check(token)):
                            word_count=word_count+1
                            #checks to see if word is already a key in the hash table/dictionary and adds it if it's not.
                            if token in story_abstract:
                                story_abstract[token]=story_abstract[token]+1
                            else:
                                story_abstract[token]=1
                    j=j+1
                #we have our word count here, now we need to transform it into its final-ish form
                keys=list(story_abstract.keys())        
                for key in keys:
                    #story_finish=[x: x*len(keys)/word_count for x in story_abstract]
                    story_abstract[key]=story_abstract[key]*len(keys)/word_count
                    cur.execute('INSERT INTO `'+story_table_name+'` (word,score) VALUES (%s,%s)',[key,story_abstract[key]])
                    con.commit()
        
            
        
    

<filename>data_cleaner.py
#<NAME>
#2019
#Lit_Guru
#Data Cleaner/Transformer
#Will reduce dimensionality in the next script
import regex as re
import MySQLdb as mdb
import os
import enchant
dictionary=enchant.Dict('en_US')
story_list=os.listdir("../Stories/")
#loop through list here
for story in story_list:
    if(story!=".DS_Store"):
        print("\n"+story,end=' ')
        url="../Stories/"+story+"/"
        con = mdb.connect('localhost', 'root', 'password', '<PASSWORD>')
        with con.cursor() as cur:
            table_check=True
            #create tables here
            #each story will get its own table
            #table will consist of each word and their scores
            story_table_name=str(story)
            story_table_name=str(re.sub("[\-'\(\)\[\]}{:;!@\*\?></\$\+\|#_,\.]",'',story_table_name))
            #story names need to be escaped with back-quotes. Thanks, Change.
            try: cur.execute('CREATE TABLE `'+story_table_name+'` (word varchar(128),score double)')
            except mdb.Error as e:
                print("Error while creating table: "+story_table_name)
                table_check=False
            if(table_check):
                story_abstract={}
                #^will contain count of each valid word
                word_count=0
                #^will count total number of words
                j=1
                while(True):
                    print(j,end=' ')
                    try: f=open(url+str(j)+".txt")
                    except FileNotFoundError:
                        break
                    chapter=f.read()
                    #removes all non alphanumerics/spaces
                    chapter=str(re.sub('[^a-zA-Z\d\s]','',chapter))
                    word_bank=chapter.split()
                    for word in word_bank:
                        token=word.lower()
                        #puts word in lowercase and then checks that it's a valid english word
                        if(dictionary.check(token)):
                            word_count=word_count+1
                            #checks to see if word is already a key in the hash table/dictionary and adds it if it's not.
                            if token in story_abstract:
                                story_abstract[token]=story_abstract[token]+1
                            else:
                                story_abstract[token]=1
                    j=j+1
                #we have our word count here, now we need to transform it into its final-ish form
                keys=list(story_abstract.keys())        
                for key in keys:
                    #story_finish=[x: x*len(keys)/word_count for x in story_abstract]
                    story_abstract[key]=story_abstract[key]*len(keys)/word_count
                    cur.execute('INSERT INTO `'+story_table_name+'` (word,score) VALUES (%s,%s)',[key,story_abstract[key]])
                    con.commit()
        
            
        
    

#<NAME> #2019 #Lit_Guru #Data Cleaner/Transformer #Will reduce dimensionality in the next script #loop through list here #create tables here #each story will get its own table #table will consist of each word and their scores #_,\.]",'',story_table_name)) #story names need to be escaped with back-quotes. Thanks, Change. #^will contain count of each valid word #^will count total number of words #removes all non alphanumerics/spaces #puts word in lowercase and then checks that it's a valid english word #checks to see if word is already a key in the hash table/dictionary and adds it if it's not. #we have our word count here, now we need to transform it into its final-ish form #story_finish=[x: x*len(keys)/word_count for x in story_abstract]