<s> import urlparse <EOL> import re <EOL> import xml . etree . ElementTree <EOL> from scrapy . spider import BaseSpider <EOL> from scrapy . selector import HtmlXPathSelector <EOL> from scrapy . http import Request <EOL> from scrapy import log <EOL> class SimpleSpider ( BaseSpider ) : <EOL> name = "<STR_LIT>" <EOL> @ staticmethod <EOL> def load_sitemap ( sitemap ) : <EOL> tree = xml . etree . ElementTree . parse ( sitemap ) <EOL> root = tree . getroot ( ) <EOL> found_urls = [ ] <EOL> for url in root : <EOL> if not url . tag . endswith ( '<STR_LIT:url>' ) : continue <EOL> for loc in url : <EOL> if not loc . tag . endswith ( '<STR_LIT>' ) : continue <EOL> found_urls . append ( loc . text ) <EOL> return found_urls <EOL> @ staticmethod <EOL> def urls_from_file ( filename ) : <EOL> f = open ( filename , '<STR_LIT:r>' ) <EOL> urls = [ l . strip ( ) for l in f ] <EOL> f . close ( ) <EOL> return urls <EOL> def __init__ ( self , urls = None , sitemap = None , mirror = None , reg_accept = None , <EOL> reg_reject = None , domains = None , url_file = None ) : <EOL> self . start_urls = [ ] <EOL> self . accept_netlocs = [ ] <EOL> self . regs_accept = None <EOL> self . regs_reject = None <EOL> if urls is not None : <EOL> urls = urls . split ( '<STR_LIT:U+002C>' ) <EOL> self . start_urls . extend ( urls ) <EOL> if url_file is not None : <EOL> self . start_urls . extend ( self . urls_from_file ( url_file ) ) <EOL> if sitemap is not None : <EOL> self . log ( "<STR_LIT>" % sitemap , log . DEBUG ) <EOL> urls = self . load_sitemap ( sitemap ) <EOL> self . start_urls . extend ( urls ) <EOL> if reg_accept is not None : <EOL> regs = reg_accept . split ( '<STR_LIT:U+002C>' ) <EOL> self . regs_accept = [ re . compile ( reg ) for reg in regs ] <EOL> self . accept_netlocs = None <EOL> if reg_reject is not None : <EOL> regs = reg_reject . split ( '<STR_LIT:U+002C>' ) <EOL> self . regs_reject = [ re . compile ( reg ) for reg in regs ] <EOL> if mirror is not None : <EOL> parts = urlparse . urlparse ( urls [ <NUM_LIT:0> ] ) <EOL> self . accept_netlocs = [ parts . netloc . lower ( ) ] <EOL> if domains is not None : <EOL> domains = domains . split ( '<STR_LIT:U+002C>' ) <EOL> self . accept_netlocs = domains <EOL> if self . accept_netlocs : <EOL> self . accept_netlocs = [ n . lower ( ) for n in self . accept_netlocs ] <EOL> self . log ( "<STR_LIT>" % self . accept_netlocs , log . DEBUG ) <EOL> if len ( self . start_urls ) > <NUM_LIT:5> : <EOL> self . log ( "<STR_LIT>" % len ( self . start_urls ) , log . DEBUG ) <EOL> else : <EOL> self . log ( "<STR_LIT>" % self . start_urls , log . DEBUG ) <EOL> """<STR_LIT>""" <EOL> @ staticmethod <EOL> def crawl_ruled ( url , accept_netlocs = None , regs_accept = None , <EOL> regs_reject = None ) : <EOL> if accept_netlocs is not None : <EOL> parts = urlparse . urlparse ( url ) <EOL> if parts . netloc . lower ( ) not in accept_netlocs : <EOL> return False <EOL> if regs_accept is not None : <EOL> accept = False <EOL> for comp in regs_accept : <EOL> if comp . search ( url ) : <EOL> accept = True <EOL> if accept is False : <EOL> return False <EOL> if regs_reject is not None : <EOL> accept = True <EOL> for comp in regs_reject : <EOL> if comp . search ( url ) : <EOL> accept = False <EOL> if accept is False : <EOL> return False <EOL> return True <EOL> """<STR_LIT>""" <EOL> def parse_css ( self , response ) : <EOL> matches = re . finditer ( '<STR_LIT>' , response . body ) <EOL> for m in matches : <EOL> rel_url = m . group ( <NUM_LIT:1> ) <EOL> abs_url = urlparse . urljoin ( response . url , rel_url . strip ( ) ) <EOL> yield Request ( abs_url , callback = self . parse_css ) <EOL> urls = re . finditer ( '<STR_LIT>' , response . body ) <EOL> ext = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> for m in urls : <EOL> rel_url = m . group ( <NUM_LIT:1> ) . strip ( "<STR_LIT:'>" ) . strip ( '<STR_LIT:">' ) <EOL> if rel_url [ - <NUM_LIT:3> : ] . lower ( ) not in ext : <EOL> continue <EOL> abs_url = urlparse . urljoin ( response . url , rel_url ) <EOL> yield Request ( abs_url , callback = lambda _ : None ) <EOL> """<STR_LIT>""" <EOL> def parse ( self , response ) : <EOL> skip_types = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> if '<STR_LIT:Content-Type>' in response . headers : <EOL> if response . headers [ '<STR_LIT:Content-Type>' ] in skip_types : <EOL> return <EOL> if response . headers [ '<STR_LIT:Content-Type>' ] . startswith ( "<STR_LIT>" ) : <EOL> return <EOL> yield Request ( urlparse . urljoin ( response . url , "<STR_LIT>" ) , <EOL> callback = lambda _ : None ) <EOL> hxs = HtmlXPathSelector ( response ) <EOL> anchors = hxs . select ( '<STR_LIT>' ) . extract ( ) <EOL> for rel_url in anchors : <EOL> abs_url = urlparse . urljoin ( response . url , rel_url . strip ( ) ) <EOL> if self . crawl_ruled ( abs_url , <EOL> accept_netlocs = self . accept_netlocs , <EOL> regs_accept = self . regs_accept , <EOL> regs_reject = self . regs_reject ) : <EOL> yield Request ( abs_url ) <EOL> image_tags = hxs . select ( '<STR_LIT>' ) . extract ( ) <EOL> for rel_url in image_tags : <EOL> abs_url = urlparse . urljoin ( response . url , rel_url . strip ( ) ) <EOL> yield Request ( abs_url , callback = lambda _ : None ) <EOL> css_urls = hxs . select ( '<STR_LIT>' ) . extract ( ) <EOL> for rel_url in css_urls : <EOL> abs_url = urlparse . urljoin ( response . url , rel_url . strip ( ) ) <EOL> yield Request ( abs_url , callback = self . parse_css ) <EOL> scripts = hxs . select ( '<STR_LIT>' ) . extract ( ) <EOL> for rel_url in scripts : <EOL> abs_url = urlparse . urljoin ( response . url , rel_url . strip ( ) ) <EOL> yield Request ( abs_url , callback = lambda _ : None ) </s>
