<s> import workerpool <EOL> import logging <EOL> import threading <EOL> from urllib2 import urlopen <EOL> from urlparse import urljoin <EOL> from Queue import Queue <EOL> from lxml import etree <EOL> from utils . xpath import build_abs_url_xpath <EOL> from utils . xpath import build_relative_url_xpath <EOL> from utils . url import is_url_in_domain <EOL> from utils . url import is_absolute <EOL> from utils . hash import url_hash <EOL> from utils . stdout import setup_stdout <EOL> setup_stdout ( ) <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> IGNORED_HREF_EXTENSIONS = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> ] <EOL> class SimpleCrawler ( object ) : <EOL> """<STR_LIT>""" <EOL> abs_url_xpath = build_abs_url_xpath ( <EOL> IGNORED_HREF_EXTENSIONS <EOL> ) <EOL> relative_url_xpath = build_relative_url_xpath ( <EOL> IGNORED_HREF_EXTENSIONS <EOL> ) <EOL> _lock = threading . Lock ( ) <EOL> def __init__ ( <EOL> self , spider , collectors = None , <EOL> post_processors = None , concurrent_requests = <NUM_LIT:1> ) : <EOL> self . pool = workerpool . WorkerPool ( size = concurrent_requests ) <EOL> self . spider = spider <EOL> self . collectors = collectors if collectors else [ ] <EOL> self . post_processors = post_processors if post_processors else [ ] <EOL> self . result = Queue ( ) <EOL> self . url_hashes = set ( ) <EOL> def start ( self ) : <EOL> """<STR_LIT>""" <EOL> more_urls = [ ] <EOL> for url in self . spider . start_urls : <EOL> _hash = url_hash ( url ) <EOL> if is_absolute ( url ) and is_url_in_domain ( <EOL> url , self . spider . domains ) and ( _hash not in self . url_hashes ) : <EOL> more_urls = more_urls + self . _crawl ( <EOL> url = url , <EOL> spider = self . spider , <EOL> url_hashes = self . url_hashes , <EOL> collectors = self . collectors , <EOL> post_processors = self . post_processors <EOL> ) <EOL> while more_urls : <EOL> _more_urls = self . pool . map ( <EOL> lambda x : self . _crawl ( <EOL> x , self . spider , self . url_hashes , self . collectors , <EOL> self . post_processors <EOL> ) , more_urls <EOL> ) <EOL> more_urls = reduce ( lambda x , y : x + y , _more_urls ) <EOL> self . pool . shutdown ( ) <EOL> self . pool . join ( ) <EOL> @ classmethod <EOL> def _crawl ( cls , url , spider , url_hashes , collectors , post_processors ) : <EOL> _blank = [ ] <EOL> with cls . _lock : <EOL> if url_hash ( url ) in url_hashes : <EOL> return _blank <EOL> else : <EOL> url_hashes . add ( url_hash ( url ) ) <EOL> try : <EOL> logger . info ( '<STR_LIT>' , url ) <EOL> response = urlopen ( url ) <EOL> except Exception as e : <EOL> logger . error ( '<STR_LIT>' , url , e ) <EOL> return _blank <EOL> try : <EOL> htmlparser = etree . HTMLParser ( ) <EOL> tree = etree . parse ( response , htmlparser ) <EOL> except Exception as e : <EOL> logger . error ( <EOL> '<STR_LIT>' , url , e ) <EOL> return _blank <EOL> try : <EOL> abs_urls = tree . xpath ( <EOL> cls . abs_url_xpath , <EOL> namespaces = { "<STR_LIT>" : "<STR_LIT>" } <EOL> ) <EOL> except Exception as e : <EOL> logger . error ( '<STR_LIT>' , url ) <EOL> abs_urls = [ ] <EOL> try : <EOL> relative_urls = tree . xpath ( <EOL> cls . relative_url_xpath , <EOL> namespaces = { "<STR_LIT>" : "<STR_LIT>" } <EOL> ) <EOL> except Exception as e : <EOL> logger . error ( '<STR_LIT>' , url ) <EOL> relative_urls = [ ] <EOL> abs_urls = filter ( <EOL> lambda x : is_url_in_domain ( x , spider . domains ) , abs_urls <EOL> ) <EOL> abs_urls = abs_urls + [ urljoin ( url , r_url ) for r_url in relative_urls ] <EOL> logger . info ( '<STR_LIT>' , len ( abs_urls ) , url ) <EOL> urls_to_crawl = [ ] <EOL> for abs_url in abs_urls : <EOL> _hash = url_hash ( abs_url ) <EOL> if _hash not in url_hashes : <EOL> urls_to_crawl . append ( abs_url ) <EOL> try : <EOL> parsed = spider . parse ( response , tree ) <EOL> except Exception as e : <EOL> logger . error ( '<STR_LIT>' , url , e ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' , url ) <EOL> try : <EOL> parsed = reduce ( <EOL> lambda x , y : y . clean ( x ) , <EOL> collectors , parsed <EOL> ) <EOL> except Exception as e : <EOL> logger . error ( '<STR_LIT>' , url , e ) <EOL> else : <EOL> try : <EOL> for post_procesor in post_processors : <EOL> post_procesor . process ( parsed ) <EOL> except Exception as e : <EOL> logger . error ( <EOL> '<STR_LIT>' , url , e ) <EOL> return urls_to_crawl </s>
