<s> """<STR_LIT>""" <EOL> import re <EOL> from ants import signals <EOL> from ants . http import Request <EOL> from ants . utils . httpobj import urlparse_cached <EOL> from ants . utils import log <EOL> class OffsiteMiddleware ( object ) : <EOL> def __init__ ( self , stats ) : <EOL> self . stats = stats <EOL> @ classmethod <EOL> def from_crawler ( cls , crawler ) : <EOL> o = cls ( crawler . stats ) <EOL> crawler . signals . connect ( o . spider_opened , signal = signals . spider_opened ) <EOL> return o <EOL> def process_spider_output ( self , response , result , spider ) : <EOL> for x in result : <EOL> if isinstance ( x , Request ) : <EOL> if x . dont_filter or self . should_follow ( x , spider ) : <EOL> yield x <EOL> else : <EOL> domain = urlparse_cached ( x ) . hostname <EOL> if domain and domain not in self . domains_seen : <EOL> self . domains_seen . add ( domain ) <EOL> log . spider_log ( "<STR_LIT>" + str ( domain ) + "<STR_LIT::>" + x . url , <EOL> level = log . DEBUG , spider = spider ) <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = spider ) <EOL> self . stats . inc_value ( '<STR_LIT>' , spider = spider ) <EOL> else : <EOL> yield x <EOL> def should_follow ( self , request , spider ) : <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> self . spider_opened ( spider ) <EOL> regex = self . host_regex <EOL> host = urlparse_cached ( request ) . hostname or '<STR_LIT>' <EOL> return bool ( regex . search ( host ) ) <EOL> def get_host_regex ( self , spider ) : <EOL> """<STR_LIT>""" <EOL> allowed_domains = getattr ( spider , '<STR_LIT>' , None ) <EOL> if not allowed_domains : <EOL> return re . compile ( '<STR_LIT>' ) <EOL> regex = r'<STR_LIT>' % '<STR_LIT:|>' . join ( re . escape ( d ) for d in allowed_domains if d is not None ) <EOL> return re . compile ( regex ) <EOL> def spider_opened ( self , spider ) : <EOL> self . host_regex = self . get_host_regex ( spider ) <EOL> self . domains_seen = set ( ) </s>
