<s> """<STR_LIT>""" <EOL> import re <EOL> from urlparse import urlparse <EOL> from ants . utils . url import url_is_from_any_domain <EOL> from ants . utils . url import canonicalize_url , url_is_from_any_domain , url_has_any_extension <EOL> from ants . utils . misc import arg_to_iter <EOL> IGNORED_EXTENSIONS = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> ] <EOL> _re_type = type ( re . compile ( "<STR_LIT>" , <NUM_LIT:0> ) ) <EOL> _matches = lambda url , regexs : any ( ( r . search ( url ) for r in regexs ) ) <EOL> _is_valid_url = lambda url : url . split ( '<STR_LIT>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] in set ( [ '<STR_LIT:http>' , '<STR_LIT>' , '<STR_LIT:file>' ] ) <EOL> class FilteringLinkExtractor ( object ) : <EOL> def __init__ ( self , link_extractor , allow , deny , allow_domains , deny_domains , <EOL> restrict_xpaths , canonicalize , deny_extensions ) : <EOL> self . link_extractor = link_extractor <EOL> self . allow_res = [ x if isinstance ( x , _re_type ) else re . compile ( x ) for x in arg_to_iter ( allow ) ] <EOL> self . deny_res = [ x if isinstance ( x , _re_type ) else re . compile ( x ) for x in arg_to_iter ( deny ) ] <EOL> self . allow_domains = set ( arg_to_iter ( allow_domains ) ) <EOL> self . deny_domains = set ( arg_to_iter ( deny_domains ) ) <EOL> self . restrict_xpaths = tuple ( arg_to_iter ( restrict_xpaths ) ) <EOL> self . canonicalize = canonicalize <EOL> if deny_extensions is None : <EOL> deny_extensions = IGNORED_EXTENSIONS <EOL> self . deny_extensions = set ( [ '<STR_LIT:.>' + e for e in arg_to_iter ( deny_extensions ) ] ) <EOL> def _link_allowed ( self , link ) : <EOL> if not _is_valid_url ( link . url ) : <EOL> return False <EOL> if self . allow_res and not _matches ( link . url , self . allow_res ) : <EOL> return False <EOL> if self . deny_res and _matches ( link . url , self . deny_res ) : <EOL> return False <EOL> parsed_url = urlparse ( link . url ) <EOL> if self . allow_domains and not url_is_from_any_domain ( parsed_url , self . allow_domains ) : <EOL> return False <EOL> if self . deny_domains and url_is_from_any_domain ( parsed_url , self . deny_domains ) : <EOL> return False <EOL> if self . deny_extensions and url_has_any_extension ( parsed_url , self . deny_extensions ) : <EOL> return False <EOL> return True <EOL> def matches ( self , url ) : <EOL> if self . allow_domains and not url_is_from_any_domain ( url , self . allow_domains ) : <EOL> return False <EOL> if self . deny_domains and url_is_from_any_domain ( url , self . deny_domains ) : <EOL> return False <EOL> allowed = [ regex . search ( url ) for regex in self . allow_res ] if self . allow_res else [ True ] <EOL> denied = [ regex . search ( url ) for regex in self . deny_res ] if self . deny_res else [ ] <EOL> return any ( allowed ) and not any ( denied ) <EOL> def _process_links ( self , links ) : <EOL> links = [ x for x in links if self . _link_allowed ( x ) ] <EOL> if self . canonicalize : <EOL> for link in links : <EOL> link . url = canonicalize_url ( urlparse ( link . url ) ) <EOL> links = self . link_extractor . _process_links ( links ) <EOL> return links <EOL> def _extract_links ( self , * args , ** kwargs ) : <EOL> return self . link_extractor . _extract_links ( * args , ** kwargs ) </s>
