<reponame>Kilenaitor/cs172
import urllib3
from bs4 import BeautifulSoup
from queue import Queue
import argparse
import threading
import sys
import os


# Disable warnings for https
urllib3.disable_warnings()

#link class has url and depth
class Link ():
    ''' Plain class to store links including the url and their depth

        depth is how far from the root the link was discovered
    '''
    def __init__(self,url,depth):
        #initialize
        self.url = url
        self.depth = depth

class Crawler():

    def __init__(self, hops, directory, num_pages):
        ''' Initialize the crawler for crawling the web
        '''
        # Tells the crawler when to stop
        self.keep_going = True
        # Hashset for visited links
        self.visited_links = set()
        # Queue to handle links
        self.links_queue = Queue()
        # Number of links visited
        self.visited = 0
        # Number of hops
        self.hops = hops
        # directory
        self.directory = directory
        # max number of pages to crawl
        self.num_pages = num_pages
        # Setup urllib 3
        self.http = urllib3.PoolManager(timeout=2.0)

    def has_visited(self, url):
        ''' Check if a link has been visited before
        '''
        return url in self.visited_links

    def enqueue_link(self, link):
        ''' Add a link to the queue externally
        '''
        self.links_queue.put(link, False)

    def start_crawling(self):
        ''' Starts crawling with specified number of threads.

            Once queue is empty, stops crawling.
        '''
        for i in range(10):
            t = threading.Thread(target=self.crawler_daemon)
            t.daemon = True
            t.start()
        self.links_queue.join()

    def crawl(self, link):
        ''' When given a link and a hop level, will parse the html for the given
            link, grab all of the hrefs, and throw them in a queue to be parsed
            later.

            Hop level is how deep the crawler will go.
            e.g. hop level of 1 means it will only parse the links found on the
            given page. hop 2 means all the links of all sub pages of the initial
            link, etc.
        '''
        # If in visited_links, return
        if self.has_visited(link.url):
            return

        # Check if we are already at max pages to crawl
        if len(self.visited_links) >= self.num_pages:
            self.keep_going = False
        # If we're done, return
        if not self.keep_going:
            return

        # Setup urllib3
        r = self.http.request('GET', link.url)
        # Decode everything with utf8 support
        html = r.data
        # Parse the document so it can be processed
        soup = BeautifulSoup(html, 'html.parser')

        # Parse and store page
        scheme = ""
        domain = ""
        endIndex = link.url.index("/",8)
        if (link.url[:7] == "http://"):
            scheme = "http://"
            domain = link.url[7:endIndex]
        else:
            scheme = "https://"
            domain = link.url[8:endIndex]

        # We only want pages that end in .edu
        if not domain.endswith('.edu'):
            return

        domainDir = "{}/{}".format(self.directory, domain)
        # If domain doesn't have a dir in output_dir, make one
        if not os.path.exists(domainDir):
            os.makedirs(domainDir)

        i = 0
        while os.path.exists("{}/{}_{}.html".format(domainDir, domain, i)):
            i += 1

        # Download to dir
        with open("{}/{}_{}.html".format(domainDir, domain, i),'w') as fid:
            fid.write(str(soup))

        # Mark the page as visited
        self.visited_links.add(link.url)

        if self.hops - link.depth <= 0:
            return

        # Adds link to the queue
        for pageLink in soup.find_all('a'):

            url = pageLink.get('href')
            if url is None:
                continue

            # Filter out or fix malformed/incomplete links
            if url.startswith('//'):
                url = scheme + domain + url[2:]
            if url.startswith('/'):
                url = scheme + domain + url[1:]

            if not url.startswith('http://'):
                continue

            #check if url is in hashset
            if url not in self.visited_links:
                self.links_queue.put(Link(url, link.depth+1), False)

    def crawler_daemon(self):
        ''' Takes a link off the parse queue, crawls and saves the page,
            and then marks the link as completed.
        '''
        while True:
            link = self.links_queue.get()
            try:
                if self.keep_going:
                    self.crawl(link)
            except Exception as e:
                print('Error: {}'.format(e))
                pass
            finally:
                self.links_queue.task_done()


def main():
    ''' Sets up the initial queue with the seeds provided and crawls the links
        until all of the levels are exhausted
    '''
    # Set up arguments parser for variables we need
    parser = argparse.ArgumentParser(prog='CS172 Web Crawler',
                                     description='Web Crawler')
    parser.add_argument('-s', '--seed-file', dest='seed_file', type=str,
                        required=True, nargs='?',
                        help='List of files to start the crawler on')
    parser.add_argument('-n', '--num-pages', dest='num_pages', type=int,
                        default=1000, required=True, nargs='?',
                        help='Total number of pages to crawl before stopping')
    parser.add_argument('-p', '--hops', dest='hops', type=int,
                        default=0, required=True, nargs='?',
                        help='How deep the crawler will go down a link path')
    parser.add_argument('-o', '--output-directory', dest='output_dir', type=str,
                        default='crawler_pages', required=True, nargs='?',
                        help=('The directory where you want the text of the'
                              'pages to be saved to')
                        )
    args = parser.parse_args()

    # Create the crawler to do the crawling
    crawler = Crawler(hops=args.hops,
                      directory=args.output_dir,
                      num_pages=args.num_pages)

    # Create output_dir if it doesn't exist
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    # Open the file and put num_pages of lines in the queue
    with open(args.seed_file, 'r') as f:
        for line in f:
            crawler.enqueue_link(Link(line.rstrip(), 1))

    # We're all set. Start crawling
    crawler.start_crawling()

    # Printing out all of the links
    print("\n".join(str(e) for e in crawler.visited_links))

if __name__ == '__main__':
    main()


import urllib3
from bs4 import BeautifulSoup
from queue import Queue
import argparse
import threading
import sys
import os


# Disable warnings for https
urllib3.disable_warnings()

#link class has url and depth
class Link ():
    ''' Plain class to store links including the url and their depth

        depth is how far from the root the link was discovered
    '''
    def __init__(self,url,depth):
        #initialize
        self.url = url
        self.depth = depth

class Crawler():

    def __init__(self, hops, directory, num_pages):
        ''' Initialize the crawler for crawling the web
        '''
        # Tells the crawler when to stop
        self.keep_going = True
        # Hashset for visited links
        self.visited_links = set()
        # Queue to handle links
        self.links_queue = Queue()
        # Number of links visited
        self.visited = 0
        # Number of hops
        self.hops = hops
        # directory
        self.directory = directory
        # max number of pages to crawl
        self.num_pages = num_pages
        # Setup urllib 3
        self.http = urllib3.PoolManager(timeout=2.0)

    def has_visited(self, url):
        ''' Check if a link has been visited before
        '''
        return url in self.visited_links

    def enqueue_link(self, link):
        ''' Add a link to the queue externally
        '''
        self.links_queue.put(link, False)

    def start_crawling(self):
        ''' Starts crawling with specified number of threads.

            Once queue is empty, stops crawling.
        '''
        for i in range(10):
            t = threading.Thread(target=self.crawler_daemon)
            t.daemon = True
            t.start()
        self.links_queue.join()

    def crawl(self, link):
        ''' When given a link and a hop level, will parse the html for the given
            link, grab all of the hrefs, and throw them in a queue to be parsed
            later.

            Hop level is how deep the crawler will go.
            e.g. hop level of 1 means it will only parse the links found on the
            given page. hop 2 means all the links of all sub pages of the initial
            link, etc.
        '''
        # If in visited_links, return
        if self.has_visited(link.url):
            return

        # Check if we are already at max pages to crawl
        if len(self.visited_links) >= self.num_pages:
            self.keep_going = False
        # If we're done, return
        if not self.keep_going:
            return

        # Setup urllib3
        r = self.http.request('GET', link.url)
        # Decode everything with utf8 support
        html = r.data
        # Parse the document so it can be processed
        soup = BeautifulSoup(html, 'html.parser')

        # Parse and store page
        scheme = ""
        domain = ""
        endIndex = link.url.index("/",8)
        if (link.url[:7] == "http://"):
            scheme = "http://"
            domain = link.url[7:endIndex]
        else:
            scheme = "https://"
            domain = link.url[8:endIndex]

        # We only want pages that end in .edu
        if not domain.endswith('.edu'):
            return

        domainDir = "{}/{}".format(self.directory, domain)
        # If domain doesn't have a dir in output_dir, make one
        if not os.path.exists(domainDir):
            os.makedirs(domainDir)

        i = 0
        while os.path.exists("{}/{}_{}.html".format(domainDir, domain, i)):
            i += 1

        # Download to dir
        with open("{}/{}_{}.html".format(domainDir, domain, i),'w') as fid:
            fid.write(str(soup))

        # Mark the page as visited
        self.visited_links.add(link.url)

        if self.hops - link.depth <= 0:
            return

        # Adds link to the queue
        for pageLink in soup.find_all('a'):

            url = pageLink.get('href')
            if url is None:
                continue

            # Filter out or fix malformed/incomplete links
            if url.startswith('//'):
                url = scheme + domain + url[2:]
            if url.startswith('/'):
                url = scheme + domain + url[1:]

            if not url.startswith('http://'):
                continue

            #check if url is in hashset
            if url not in self.visited_links:
                self.links_queue.put(Link(url, link.depth+1), False)

    def crawler_daemon(self):
        ''' Takes a link off the parse queue, crawls and saves the page,
            and then marks the link as completed.
        '''
        while True:
            link = self.links_queue.get()
            try:
                if self.keep_going:
                    self.crawl(link)
            except Exception as e:
                print('Error: {}'.format(e))
                pass
            finally:
                self.links_queue.task_done()


def main():
    ''' Sets up the initial queue with the seeds provided and crawls the links
        until all of the levels are exhausted
    '''
    # Set up arguments parser for variables we need
    parser = argparse.ArgumentParser(prog='CS172 Web Crawler',
                                     description='Web Crawler')
    parser.add_argument('-s', '--seed-file', dest='seed_file', type=str,
                        required=True, nargs='?',
                        help='List of files to start the crawler on')
    parser.add_argument('-n', '--num-pages', dest='num_pages', type=int,
                        default=1000, required=True, nargs='?',
                        help='Total number of pages to crawl before stopping')
    parser.add_argument('-p', '--hops', dest='hops', type=int,
                        default=0, required=True, nargs='?',
                        help='How deep the crawler will go down a link path')
    parser.add_argument('-o', '--output-directory', dest='output_dir', type=str,
                        default='crawler_pages', required=True, nargs='?',
                        help=('The directory where you want the text of the'
                              'pages to be saved to')
                        )
    args = parser.parse_args()

    # Create the crawler to do the crawling
    crawler = Crawler(hops=args.hops,
                      directory=args.output_dir,
                      num_pages=args.num_pages)

    # Create output_dir if it doesn't exist
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    # Open the file and put num_pages of lines in the queue
    with open(args.seed_file, 'r') as f:
        for line in f:
            crawler.enqueue_link(Link(line.rstrip(), 1))

    # We're all set. Start crawling
    crawler.start_crawling()

    # Printing out all of the links
    print("\n".join(str(e) for e in crawler.visited_links))

if __name__ == '__main__':
    main()
# Disable warnings for https #link class has url and depth Plain class to store links including the url and their depth          depth is how far from the root the link was discovered #initialize Initialize the crawler for crawling the web # Tells the crawler when to stop # Hashset for visited links # Queue to handle links # Number of links visited # Number of hops # directory # max number of pages to crawl # Setup urllib 3 Check if a link has been visited before Add a link to the queue externally Starts crawling with specified number of threads.              Once queue is empty, stops crawling. When given a link and a hop level, will parse the html for the given             link, grab all of the hrefs, and throw them in a queue to be parsed             later.              Hop level is how deep the crawler will go.             e.g. hop level of 1 means it will only parse the links found on the             given page. hop 2 means all the links of all sub pages of the initial             link, etc. # If in visited_links, return # Check if we are already at max pages to crawl # If we're done, return # Setup urllib3 # Decode everything with utf8 support # Parse the document so it can be processed # Parse and store page # We only want pages that end in .edu # If domain doesn't have a dir in output_dir, make one # Download to dir # Mark the page as visited # Adds link to the queue # Filter out or fix malformed/incomplete links #check if url is in hashset Takes a link off the parse queue, crawls and saves the page,             and then marks the link as completed. Sets up the initial queue with the seeds provided and crawls the links         until all of the levels are exhausted # Set up arguments parser for variables we need # Create the crawler to do the crawling # Create output_dir if it doesn't exist # Open the file and put num_pages of lines in the queue # We're all set. Start crawling # Printing out all of the links