<s> from common_crawl . base import CommonCrawlJob <EOL> from bs4 import BeautifulSoup , SoupStrainer <EOL> only_a_tags = SoupStrainer ( "<STR_LIT:a>" ) <EOL> class RegexLinksJob ( CommonCrawlJob ) : <EOL> def configure_options ( self ) : <EOL> super ( RegixLinksJob , self ) . configure_options ( ) <EOL> self . add_passthrough_option ( '<STR_LIT>' ) <EOL> self . add_passthrough_option ( '<STR_LIT>' , default = None ) <EOL> self . add_passthrough_option ( '<STR_LIT>' , default = None ) <EOL> def mapper_init ( self ) : <EOL> if not self . options . pattern : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> regex_flags = re . I if self . options . insensitive is not None else None <EOL> self . link_pattern = re . compile ( self . options . pattern , regex_flags ) <EOL> self . exact = self . options . exact is not None <EOL> self . match_func = self . link_pattern . search if not self . exact else self . link_pattern . match <EOL> def parse_content ( self , content ) : <EOL> return BeautifulSoup ( content , '<STR_LIT>' , parse_only = only_a_tags ) <EOL> def process_html ( self , url , headers , content , soup ) : <EOL> for tag in soup : <EOL> if '<STR_LIT>' not in tag . attrs : <EOL> continue <EOL> href = tag . attrs [ '<STR_LIT>' ] . strip ( ) <EOL> norm_href = urlparse . urljoin ( url , href ) <EOL> if self . match_func ( norm_href ) : <EOL> yield norm_href , <NUM_LIT:1> <EOL> def reducer ( self , key , values ) : <EOL> yield key , sum ( values ) <EOL> combiner = reducer <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> RegexLinksJob . run ( ) </s>
