<s> """<STR_LIT>""" <EOL> __title__ = '<STR_LIT>' <EOL> __author__ = '<STR_LIT>' <EOL> __license__ = '<STR_LIT>' <EOL> __copyright__ = '<STR_LIT>' <EOL> import logging <EOL> import re <EOL> from urllib . parse import parse_qs , urljoin , urlparse , urlsplit , urlunsplit <EOL> from tldextract import tldextract <EOL> log = logging . getLogger ( __name__ ) <EOL> MAX_FILE_MEMO = <NUM_LIT> <EOL> DATE_REGEX = r'<STR_LIT>' <EOL> ALLOWED_TYPES = [ '<STR_LIT:html>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> GOOD_PATHS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT:v>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> BAD_CHUNKS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:info>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> BAD_DOMAINS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> def remove_args ( url , keep_params = ( ) , frags = False ) : <EOL> """<STR_LIT>""" <EOL> parsed = urlsplit ( url ) <EOL> filtered_query = '<STR_LIT:&>' . join ( <EOL> qry_item for qry_item in parsed . query . split ( '<STR_LIT:&>' ) <EOL> if qry_item . startswith ( keep_params ) <EOL> ) <EOL> if frags : <EOL> frag = parsed [ <NUM_LIT:4> : ] <EOL> else : <EOL> frag = ( '<STR_LIT>' , ) <EOL> return urlunsplit ( parsed [ : <NUM_LIT:3> ] + ( filtered_query , ) + frag ) <EOL> def redirect_back ( url , source_domain ) : <EOL> """<STR_LIT>""" <EOL> parse_data = urlparse ( url ) <EOL> domain = parse_data . netloc <EOL> query = parse_data . query <EOL> if source_domain in domain or domain in source_domain : <EOL> return url <EOL> query_item = parse_qs ( query ) <EOL> if query_item . get ( '<STR_LIT:url>' ) : <EOL> return query_item [ '<STR_LIT:url>' ] [ <NUM_LIT:0> ] <EOL> return url <EOL> def prepare_url ( url , source_url = None ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> if source_url is not None : <EOL> source_domain = urlparse ( source_url ) . netloc <EOL> proper_url = urljoin ( source_url , url ) <EOL> proper_url = redirect_back ( proper_url , source_domain ) <EOL> else : <EOL> proper_url = url <EOL> except ValueError as e : <EOL> log . critical ( '<STR_LIT>' % ( url , str ( e ) ) ) <EOL> proper_url = '<STR_LIT>' <EOL> return proper_url <EOL> def valid_url ( url , verbose = False , test = False ) : <EOL> """<STR_LIT>""" <EOL> if test : <EOL> url = prepare_url ( url ) <EOL> if url is None or len ( url ) < <NUM_LIT:11> : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> r1 = ( '<STR_LIT>' in url ) <EOL> r2 = ( '<STR_LIT>' not in url ) and ( '<STR_LIT>' not in url ) <EOL> if r1 or r2 : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> path = urlparse ( url ) . path <EOL> if not path . startswith ( '<STR_LIT:/>' ) : <EOL> return False <EOL> if path . endswith ( '<STR_LIT:/>' ) : <EOL> path = path [ : - <NUM_LIT:1> ] <EOL> path_chunks = [ x for x in path . split ( '<STR_LIT:/>' ) if len ( x ) > <NUM_LIT:0> ] <EOL> if len ( path_chunks ) > <NUM_LIT:0> : <EOL> file_type = url_to_filetype ( url ) <EOL> if file_type and file_type not in ALLOWED_TYPES : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> last_chunk = path_chunks [ - <NUM_LIT:1> ] . split ( '<STR_LIT:.>' ) <EOL> if len ( last_chunk ) > <NUM_LIT:1> : <EOL> path_chunks [ - <NUM_LIT:1> ] = last_chunk [ - <NUM_LIT:2> ] <EOL> if '<STR_LIT:index>' in path_chunks : <EOL> path_chunks . remove ( '<STR_LIT:index>' ) <EOL> tld_dat = tldextract . extract ( url ) <EOL> subd = tld_dat . subdomain <EOL> tld = tld_dat . domain . lower ( ) <EOL> url_slug = path_chunks [ - <NUM_LIT:1> ] if path_chunks else '<STR_LIT>' <EOL> if tld in BAD_DOMAINS : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> if len ( path_chunks ) == <NUM_LIT:0> : <EOL> dash_count , underscore_count = <NUM_LIT:0> , <NUM_LIT:0> <EOL> else : <EOL> dash_count = url_slug . count ( '<STR_LIT:->' ) <EOL> underscore_count = url_slug . count ( '<STR_LIT:_>' ) <EOL> if url_slug and ( dash_count > <NUM_LIT:4> or underscore_count > <NUM_LIT:4> ) : <EOL> if dash_count >= underscore_count : <EOL> if tld not in [ x . lower ( ) for x in url_slug . split ( '<STR_LIT:->' ) ] : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return True <EOL> if underscore_count > dash_count : <EOL> if tld not in [ x . lower ( ) for x in url_slug . split ( '<STR_LIT:_>' ) ] : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return True <EOL> if len ( path_chunks ) <= <NUM_LIT:1> : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> for b in BAD_CHUNKS : <EOL> if b in path_chunks or b == subd : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> match_date = re . search ( DATE_REGEX , url ) <EOL> if match_date is not None : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return True <EOL> for GOOD in GOOD_PATHS : <EOL> if GOOD . lower ( ) in [ p . lower ( ) for p in path_chunks ] : <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return True <EOL> if verbose : print ( '<STR_LIT>' % url ) <EOL> return False <EOL> def url_to_filetype ( abs_url ) : <EOL> """<STR_LIT>""" <EOL> path = urlparse ( abs_url ) . path <EOL> if path . endswith ( '<STR_LIT:/>' ) : <EOL> path = path [ : - <NUM_LIT:1> ] <EOL> path_chunks = [ x for x in path . split ( '<STR_LIT:/>' ) if len ( x ) > <NUM_LIT:0> ] <EOL> last_chunk = path_chunks [ - <NUM_LIT:1> ] . split ( '<STR_LIT:.>' ) <EOL> if len ( last_chunk ) < <NUM_LIT:2> : <EOL> return None <EOL> file_type = last_chunk [ - <NUM_LIT:1> ] <EOL> if len ( file_type ) <= <NUM_LIT:5> or file_type . lower ( ) in ALLOWED_TYPES : <EOL> return file_type . lower ( ) <EOL> return None <EOL> def get_domain ( abs_url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if abs_url is None : <EOL> return None <EOL> return urlparse ( abs_url , ** kwargs ) . netloc <EOL> def get_scheme ( abs_url , ** kwargs ) : <EOL> """<STR_LIT:U+0020>""" <EOL> if abs_url is None : <EOL> return None <EOL> return urlparse ( abs_url , ** kwargs ) . scheme <EOL> def get_path ( abs_url , ** kwargs ) : <EOL> """<STR_LIT:U+0020>""" <EOL> if abs_url is None : <EOL> return None <EOL> return urlparse ( abs_url , ** kwargs ) . path <EOL> def is_abs_url ( url ) : <EOL> """<STR_LIT>""" <EOL> regex = re . compile ( <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' , re . IGNORECASE ) <EOL> c_regex = re . compile ( regex ) <EOL> return ( c_regex . search ( url ) is not None ) </s>
