<s> """<STR_LIT>""" <EOL> import time <EOL> from six . moves . urllib . parse import urlencode <EOL> from scrapy . spiders import Spider <EOL> from scrapy . http import Request <EOL> from scrapy . item import Item <EOL> from scrapy . linkextractors import LinkExtractor <EOL> class MetaSpider ( Spider ) : <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , * args , ** kwargs ) : <EOL> super ( MetaSpider , self ) . __init__ ( * args , ** kwargs ) <EOL> self . meta = { } <EOL> def closed ( self , reason ) : <EOL> self . meta [ '<STR_LIT>' ] = reason <EOL> class FollowAllSpider ( MetaSpider ) : <EOL> name = '<STR_LIT>' <EOL> link_extractor = LinkExtractor ( ) <EOL> def __init__ ( self , total = <NUM_LIT:10> , show = <NUM_LIT:20> , order = "<STR_LIT>" , maxlatency = <NUM_LIT:0.0> , * args , ** kwargs ) : <EOL> super ( FollowAllSpider , self ) . __init__ ( * args , ** kwargs ) <EOL> self . urls_visited = [ ] <EOL> self . times = [ ] <EOL> qargs = { '<STR_LIT>' : total , '<STR_LIT>' : show , '<STR_LIT>' : order , '<STR_LIT>' : maxlatency } <EOL> url = "<STR_LIT>" % urlencode ( qargs , doseq = <NUM_LIT:1> ) <EOL> self . start_urls = [ url ] <EOL> def parse ( self , response ) : <EOL> self . urls_visited . append ( response . url ) <EOL> self . times . append ( time . time ( ) ) <EOL> for link in self . link_extractor . extract_links ( response ) : <EOL> yield Request ( link . url , callback = self . parse ) <EOL> class DelaySpider ( MetaSpider ) : <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , n = <NUM_LIT:1> , b = <NUM_LIT:0> , * args , ** kwargs ) : <EOL> super ( DelaySpider , self ) . __init__ ( * args , ** kwargs ) <EOL> self . n = n <EOL> self . b = b <EOL> self . t1 = self . t2 = self . t2_err = <NUM_LIT:0> <EOL> def start_requests ( self ) : <EOL> self . t1 = time . time ( ) <EOL> url = "<STR_LIT>" % ( self . n , self . b ) <EOL> yield Request ( url , callback = self . parse , errback = self . errback ) <EOL> def parse ( self , response ) : <EOL> self . t2 = time . time ( ) <EOL> def errback ( self , failure ) : <EOL> self . t2_err = time . time ( ) <EOL> class SimpleSpider ( MetaSpider ) : <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , url = "<STR_LIT>" , * args , ** kwargs ) : <EOL> super ( SimpleSpider , self ) . __init__ ( * args , ** kwargs ) <EOL> self . start_urls = [ url ] <EOL> def parse ( self , response ) : <EOL> self . logger . info ( "<STR_LIT>" % response . status ) <EOL> class ItemSpider ( FollowAllSpider ) : <EOL> name = '<STR_LIT>' <EOL> def parse ( self , response ) : <EOL> for request in super ( ItemSpider , self ) . parse ( response ) : <EOL> yield request <EOL> yield Item ( ) <EOL> yield { } <EOL> class DefaultError ( Exception ) : <EOL> pass <EOL> class ErrorSpider ( FollowAllSpider ) : <EOL> name = '<STR_LIT:error>' <EOL> exception_cls = DefaultError <EOL> def raise_exception ( self ) : <EOL> raise self . exception_cls ( '<STR_LIT>' ) <EOL> def parse ( self , response ) : <EOL> for request in super ( ErrorSpider , self ) . parse ( response ) : <EOL> yield request <EOL> self . raise_exception ( ) <EOL> class BrokenStartRequestsSpider ( FollowAllSpider ) : <EOL> fail_before_yield = False <EOL> fail_yielding = False <EOL> def __init__ ( self , * a , ** kw ) : <EOL> super ( BrokenStartRequestsSpider , self ) . __init__ ( * a , ** kw ) <EOL> self . seedsseen = [ ] <EOL> def start_requests ( self ) : <EOL> if self . fail_before_yield : <EOL> <NUM_LIT:1> / <NUM_LIT:0> <EOL> for s in range ( <NUM_LIT:100> ) : <EOL> qargs = { '<STR_LIT>' : <NUM_LIT:10> , '<STR_LIT>' : s } <EOL> url = "<STR_LIT>" % urlencode ( qargs , doseq = <NUM_LIT:1> ) <EOL> yield Request ( url , meta = { '<STR_LIT>' : s } ) <EOL> if self . fail_yielding : <EOL> <NUM_LIT:2> / <NUM_LIT:0> <EOL> assert self . seedsseen , '<STR_LIT>' <EOL> def parse ( self , response ) : <EOL> self . seedsseen . append ( response . meta . get ( '<STR_LIT>' ) ) <EOL> for req in super ( BrokenStartRequestsSpider , self ) . parse ( response ) : <EOL> yield req <EOL> class SingleRequestSpider ( MetaSpider ) : <EOL> seed = None <EOL> callback_func = None <EOL> errback_func = None <EOL> def start_requests ( self ) : <EOL> if isinstance ( self . seed , Request ) : <EOL> yield self . seed . replace ( callback = self . parse , errback = self . on_error ) <EOL> else : <EOL> yield Request ( self . seed , callback = self . parse , errback = self . on_error ) <EOL> def parse ( self , response ) : <EOL> self . meta . setdefault ( '<STR_LIT>' , [ ] ) . append ( response ) <EOL> if callable ( self . callback_func ) : <EOL> return self . callback_func ( response ) <EOL> if '<STR_LIT>' in response . meta : <EOL> return response . meta [ '<STR_LIT>' ] <EOL> def on_error ( self , failure ) : <EOL> self . meta [ '<STR_LIT>' ] = failure <EOL> if callable ( self . errback_func ) : <EOL> return self . errback_func ( failure ) <EOL> class DuplicateStartRequestsSpider ( Spider ) : <EOL> dont_filter = True <EOL> name = '<STR_LIT>' <EOL> distinct_urls = <NUM_LIT:2> <EOL> dupe_factor = <NUM_LIT:3> <EOL> def start_requests ( self ) : <EOL> for i in range ( <NUM_LIT:0> , self . distinct_urls ) : <EOL> for j in range ( <NUM_LIT:0> , self . dupe_factor ) : <EOL> url = "<STR_LIT>" % i <EOL> yield self . make_requests_from_url ( url ) <EOL> def make_requests_from_url ( self , url ) : <EOL> return Request ( url , dont_filter = self . dont_filter ) <EOL> def __init__ ( self , url = "<STR_LIT>" , * args , ** kwargs ) : <EOL> super ( DuplicateStartRequestsSpider , self ) . __init__ ( * args , ** kwargs ) <EOL> self . visited = <NUM_LIT:0> <EOL> def parse ( self , response ) : <EOL> self . visited += <NUM_LIT:1> </s>
