<reponame>rexarski/ggtown-ds
#conda install -c anaconda nltk
#inside python run: import nltk; 
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
#nltk.download('wordnet')
#nltk.download('stopwords')

#---------------------------
#NOTES
#---------------------------

#NOTE: string.printable will give the all sets of punctuation, 
#	   digits, ascii_letters and whitespace.


import nltk
import string

from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords
from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize

import os

#USER PARAM
input_path='DRACULA.txt'

#OUTPUT FILE
output='transactions.txt'
if os.path.exists(output): os.remove(output)

#INITIALIZE
lemmatizer 	= 	WordNetLemmatizer()
ps 			=	PorterStemmer()

#ADD MORE
stopwords	=	stopwords.words('english')
stopwords.append('mr')
stopwords.append('mrs')
stopwords.append('wa')
stopwords.append('dr')
stopwords.append('said')
# print(stopwords)

def read_and_clean(path,START=0,STOP=-1):

	#READ IN AS ONE BIG STING
	file = open(path, 'rt')
	text = file.read().lower()	#CONVERT TO LOWER CASE
	file.close()

	#MAKE INTO NLTK OBJECT AND SEARCH TEXT 
	nltk_object=nltk.Text(word_tokenize(text))
	print(nltk_object.concordance("dracula")) 	#SEARCH TEXT
	nltk_object.dispersion_plot(["dracula", "jonathan","mina"])

	#REMOVE HEADER, AND NEW LINES
	lines = text.splitlines(); text=''; 
	lines=lines[START:STOP]    # mystring.replace('\n', ' ')
	for line in lines: text=text+' '+line

	#ONLY KEEP CHAR IN PRINTABLE
	tmp=''; printable = set(string.printable); # print(printable)
	for char in text:
		if(char in printable): tmp=tmp+char
	text=tmp

	#BREAK INTO SENTENCES
	sentences=nltk.tokenize.sent_tokenize(text)
	print("NUMBER OF SENTENCES FOUND:",len(sentences))

	#CLEAN SENTENCES AND LEMMATIZE
	keep='0123456789abcdefghijklmnopqrstuvwxy';

	new_sentences=[]; vocabulary=[]
	for sentence in sentences:
		new_sentence=''

		for word in sentence.split():
			word=word.replace("'",'') #wasn't to wasnt
			
			#ONLY KEEP CHAR IN "keep"
			tmp2=''
			for char in word: 
				if(char in keep): 
					tmp2=tmp2+char
				else:
					tmp2=tmp2+' '

			#STEM OF LEMMATIZE
			word=tmp2
			new_word=lemmatizer.lemmatize(word) #LEMMATIZATION

			#REMOVE WHITE SPACES
			new_word=new_word.replace(' ', '')

			#BUILD NEW SENTANCE BACK UP
			if( new_word not in stopwords):
				if(new_sentence==''):
					new_sentence=new_word
				else:
					new_sentence=new_sentence+','+new_word
				if(new_word not in vocabulary): vocabulary.append(new_word)

		#SAVE SENTANCE TO OUTPUT FILE
		if(len(new_sentence.split(','))>2):
			f = open(output, "a")
			f.write(new_sentence+"\n")
			f.close()

	print("VOCAB LENGTH",len(vocabulary))



#RUN FUNCTION
read_and_clean(input_path,400,-400)

#conda install -c anaconda nltk
#inside python run: import nltk; 
#nltk.download('punkt')
#nltk.download('averaged_perceptron_tagger')
#nltk.download('wordnet')
#nltk.download('stopwords')

#---------------------------
#NOTES
#---------------------------

#NOTE: string.printable will give the all sets of punctuation, 
#	   digits, ascii_letters and whitespace.


import nltk
import string

from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords
from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize

import os

#USER PARAM
input_path='DRACULA.txt'

#OUTPUT FILE
output='transactions.txt'
if os.path.exists(output): os.remove(output)

#INITIALIZE
lemmatizer 	= 	WordNetLemmatizer()
ps 			=	PorterStemmer()

#ADD MORE
stopwords	=	stopwords.words('english')
stopwords.append('mr')
stopwords.append('mrs')
stopwords.append('wa')
stopwords.append('dr')
stopwords.append('said')
# print(stopwords)

def read_and_clean(path,START=0,STOP=-1):

	#READ IN AS ONE BIG STING
	file = open(path, 'rt')
	text = file.read().lower()	#CONVERT TO LOWER CASE
	file.close()

	#MAKE INTO NLTK OBJECT AND SEARCH TEXT 
	nltk_object=nltk.Text(word_tokenize(text))
	print(nltk_object.concordance("dracula")) 	#SEARCH TEXT
	nltk_object.dispersion_plot(["dracula", "jonathan","mina"])

	#REMOVE HEADER, AND NEW LINES
	lines = text.splitlines(); text=''; 
	lines=lines[START:STOP]    # mystring.replace('\n', ' ')
	for line in lines: text=text+' '+line

	#ONLY KEEP CHAR IN PRINTABLE
	tmp=''; printable = set(string.printable); # print(printable)
	for char in text:
		if(char in printable): tmp=tmp+char
	text=tmp

	#BREAK INTO SENTENCES
	sentences=nltk.tokenize.sent_tokenize(text)
	print("NUMBER OF SENTENCES FOUND:",len(sentences))

	#CLEAN SENTENCES AND LEMMATIZE
	keep='0123456789abcdefghijklmnopqrstuvwxy';

	new_sentences=[]; vocabulary=[]
	for sentence in sentences:
		new_sentence=''

		for word in sentence.split():
			word=word.replace("'",'') #wasn't to wasnt
			
			#ONLY KEEP CHAR IN "keep"
			tmp2=''
			for char in word: 
				if(char in keep): 
					tmp2=tmp2+char
				else:
					tmp2=tmp2+' '

			#STEM OF LEMMATIZE
			word=tmp2
			new_word=lemmatizer.lemmatize(word) #LEMMATIZATION

			#REMOVE WHITE SPACES
			new_word=new_word.replace(' ', '')

			#BUILD NEW SENTANCE BACK UP
			if( new_word not in stopwords):
				if(new_sentence==''):
					new_sentence=new_word
				else:
					new_sentence=new_sentence+','+new_word
				if(new_word not in vocabulary): vocabulary.append(new_word)

		#SAVE SENTANCE TO OUTPUT FILE
		if(len(new_sentence.split(','))>2):
			f = open(output, "a")
			f.write(new_sentence+"\n")
			f.close()

	print("VOCAB LENGTH",len(vocabulary))



#RUN FUNCTION
read_and_clean(input_path,400,-400)
#conda install -c anaconda nltk #inside python run: import nltk; #nltk.download('punkt') #nltk.download('averaged_perceptron_tagger') #nltk.download('wordnet') #nltk.download('stopwords') #--------------------------- #NOTES #--------------------------- #NOTE: string.printable will give the all sets of punctuation, #	   digits, ascii_letters and whitespace. #USER PARAM #OUTPUT FILE #INITIALIZE #ADD MORE # print(stopwords) #READ IN AS ONE BIG STING #CONVERT TO LOWER CASE #MAKE INTO NLTK OBJECT AND SEARCH TEXT #SEARCH TEXT #REMOVE HEADER, AND NEW LINES # mystring.replace('\n', ' ') #ONLY KEEP CHAR IN PRINTABLE # print(printable) #BREAK INTO SENTENCES #CLEAN SENTENCES AND LEMMATIZE #wasn't to wasnt #ONLY KEEP CHAR IN "keep" #STEM OF LEMMATIZE #LEMMATIZATION #REMOVE WHITE SPACES #BUILD NEW SENTANCE BACK UP #SAVE SENTANCE TO OUTPUT FILE #RUN FUNCTION