import requests
from bs4 import BeautifulSoup
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer

#Code for cleaning the string was acquired and altered for use in this project
#The original webpage and more information about this can be found at
#https://machinelearningmastery.com/clean-text-machine-learning-python/
def clean_string(string):
    #Split the block of text from website into seperate tokens
    tokens = word_tokenize(string)
    #Clean out the punctuation and other misc tokens
    words = [word for word in tokens if word.isalpha()]
    #Normalize the words to all lowercase
    words = [word.lower() for word in words]
    #Filter out stopwords
    stop_words = set(stopwords.words('english'))
    words = [w for w in words if not w in stop_words]
    #Stemming of the words to their base forms
    porter = PorterStemmer()
    stemmed = [porter.stem(word) for word in words]
    stemmed.sort()
    return stemmed

#Information about how to install and use BeautifulSoup4 was aquired from
#https://medium.freecodecamp.org/how-to-scrape-websites-with-python-and-beautifulsoup-5946935d93fe
def web_scraper(url_in):
    page = requests.get(url_in)
    soup = BeautifulSoup(page.content, 'html.parser')
    title = soup.title.text
    body_text = ""
    for body in soup.find_all('p'):
        body_text += " " + body.text
    body_text = clean_string(body_text)
    return(title, body_text)

import requests
from bs4 import BeautifulSoup
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer

#Code for cleaning the string was acquired and altered for use in this project
#The original webpage and more information about this can be found at
#https://machinelearningmastery.com/clean-text-machine-learning-python/
def clean_string(string):
    #Split the block of text from website into seperate tokens
    tokens = word_tokenize(string)
    #Clean out the punctuation and other misc tokens
    words = [word for word in tokens if word.isalpha()]
    #Normalize the words to all lowercase
    words = [word.lower() for word in words]
    #Filter out stopwords
    stop_words = set(stopwords.words('english'))
    words = [w for w in words if not w in stop_words]
    #Stemming of the words to their base forms
    porter = PorterStemmer()
    stemmed = [porter.stem(word) for word in words]
    stemmed.sort()
    return stemmed

#Information about how to install and use BeautifulSoup4 was aquired from
#https://medium.freecodecamp.org/how-to-scrape-websites-with-python-and-beautifulsoup-5946935d93fe
def web_scraper(url_in):
    page = requests.get(url_in)
    soup = BeautifulSoup(page.content, 'html.parser')
    title = soup.title.text
    body_text = ""
    for body in soup.find_all('p'):
        body_text += " " + body.text
    body_text = clean_string(body_text)
    return(title, body_text)

#Code for cleaning the string was acquired and altered for use in this project #The original webpage and more information about this can be found at #https://machinelearningmastery.com/clean-text-machine-learning-python/ #Split the block of text from website into seperate tokens #Clean out the punctuation and other misc tokens #Normalize the words to all lowercase #Filter out stopwords #Stemming of the words to their base forms #Information about how to install and use BeautifulSoup4 was aquired from #https://medium.freecodecamp.org/how-to-scrape-websites-with-python-and-beautifulsoup-5946935d93fe