<s> from common_crawl . base import CommonCrawlJob <EOL> from bs4 import BeautifulSoup , SoupStrainer <EOL> only_a_tags = SoupStrainer ( "<STR_LIT:a>" ) <EOL> def parse_file_ext_from_url ( url ) : <EOL> parsed = urlparse . urlparse ( url ) <EOL> last_path = parsed . path . split ( '<STR_LIT:/>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> if '<STR_LIT:.>' in last_path : <EOL> extension = last_path . split ( '<STR_LIT:.>' ) [ - <NUM_LIT:1> ] . strip ( ) <EOL> if extension : <EOL> return extension <EOL> return None <EOL> class RegexURLJob ( CommonCrawlJob ) : <EOL> OUTPUT_PROTOCOL = RawValueProtocol <EOL> def configure_options ( self ) : <EOL> super ( RegexURLJob , self ) . configure_options ( ) <EOL> self . add_passthrough_option ( '<STR_LIT>' ) <EOL> self . add_passthrough_option ( '<STR_LIT>' , default = None ) <EOL> self . add_passthrough_option ( '<STR_LIT>' , default = None ) <EOL> def mapper_init ( self ) : <EOL> if not self . options . pattern : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> regex_flags = re . I if self . options . insensitive is not None else None <EOL> self . link_pattern = re . compile ( self . options . pattern , regex_flags ) <EOL> self . exact = self . options . exact is not None <EOL> self . match_func = self . link_pattern . search if not self . exact else self . link_pattern . match <EOL> def mapper ( self , record ) : <EOL> url = record . url <EOL> if self . match_func ( url ) : <EOL> yield url <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> RegexURLJob . run ( ) </s>
