<s> import re <EOL> try : <EOL> from bs4 import BeautifulSoup , SoupStrainer <EOL> except Exception : <EOL> from BeautifulSoup import BeautifulSoup , SoupStrainer <EOL> from cabu . exceptions import LinkExtractorException <EOL> def extract_links ( response_content , unique = False , blacklist_domains = [ ] , <EOL> whitelist_domains = [ ] , regex = None , zen_path = None , <EOL> blacklist_extensions = [ ] , whitelist_extensions = [ ] ) : <EOL> """<STR_LIT>""" <EOL> if any ( [ item in blacklist_domains for item in whitelist_domains ] ) or any ( [ item in blacklist_extensions for item in whitelist_extensions ] ) : <EOL> raise LinkExtractorException ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> soup = BeautifulSoup ( <EOL> response_content , "<STR_LIT>" , parse_only = SoupStrainer ( '<STR_LIT:a>' ) <EOL> ) <EOL> links = [ a . text for a in soup ] <EOL> if unique : <EOL> links = list ( set ( links ) ) <EOL> if regex : <EOL> links = filter_links ( links , regex ) <EOL> if whitelist_domains : <EOL> for domn in whitelist_domains : <EOL> links = filter_links ( links , domn . replace ( '<STR_LIT:.>' , '<STR_LIT>' ) , include = True ) <EOL> if blacklist_domains : <EOL> for domn in blacklist_domains : <EOL> links = filter_links ( links , domn . replace ( '<STR_LIT:.>' , '<STR_LIT>' ) , include = False ) <EOL> if whitelist_extensions : <EOL> for ext in whitelist_extensions : <EOL> links = filter_links ( links , ext . replace ( '<STR_LIT:.>' , '<STR_LIT>' ) , include = True ) <EOL> if blacklist_extensions : <EOL> for ext in blacklist_extensions : <EOL> links = filter_links ( links , ext . replace ( '<STR_LIT:.>' , '<STR_LIT>' ) , include = False ) <EOL> return links <EOL> def filter_links ( links , regex , include = True ) : <EOL> """<STR_LIT>""" <EOL> links_filtered = [ ] <EOL> for link in links : <EOL> if re . search ( regex , link ) and include : <EOL> links_filtered . append ( link ) <EOL> elif not re . search ( regex , link ) and not include : <EOL> links_filtered . append ( link ) <EOL> return links_filtered </s>
