<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import , division , print_function <EOL> import logging <EOL> import re <EOL> import sys <EOL> from itertools import groupby <EOL> import pylru <EOL> from iso3166 import countries <EOL> from six import iteritems , itervalues , PY3 , string_types , text_type <EOL> from six . moves . urllib . parse import urlparse , parse_qs , ParseResult <EOL> try : <EOL> import pkg_resources <EOL> except ImportError : <EOL> import os <EOL> class pkg_resources ( object ) : <EOL> """<STR_LIT>""" <EOL> @ classmethod <EOL> def resource_stream ( cls , package , resource_name ) : <EOL> moddir = os . path . dirname ( __file__ ) <EOL> f = os . path . join ( moddir , resource_name ) <EOL> return open ( f ) <EOL> try : <EOL> import cPickle as pickle <EOL> except ImportError : <EOL> import pickle <EOL> __all__ = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> _country_codes = [ country . alpha2 . lower ( ) <EOL> for country in countries ] <EOL> _country_codes += [ '<STR_LIT>' ] <EOL> _domain_cache = pylru . lrucache ( <NUM_LIT> ) <EOL> _naive_re = re . compile ( r'<STR_LIT>' ) <EOL> _naive_params = ( '<STR_LIT:q>' , '<STR_LIT>' , '<STR_LIT:k>' , '<STR_LIT>' , '<STR_LIT>' , ) <EOL> def _unicode_parse_qs ( qs , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if PY3 or isinstance ( qs , bytes ) : <EOL> return parse_qs ( qs , ** kwargs ) <EOL> qs = qs . encode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> query = parse_qs ( qs , ** kwargs ) <EOL> unicode_query = { } <EOL> for key in query : <EOL> uni_key = key . decode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) <EOL> if uni_key == '<STR_LIT>' : <EOL> continue <EOL> unicode_query [ uni_key ] = [ p . decode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) for p in query [ key ] ] <EOL> return unicode_query <EOL> def _unicode_urlparse ( url , encoding = '<STR_LIT:utf-8>' , errors = '<STR_LIT:ignore>' ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( url , bytes ) : <EOL> url = url . decode ( encoding , errors ) <EOL> elif isinstance ( url , ParseResult ) : <EOL> parts = list ( url ) <EOL> for i in range ( len ( parts ) ) : <EOL> if isinstance ( parts [ i ] , bytes ) : <EOL> parts [ i ] = parts [ i ] . decode ( encoding , errors ) <EOL> return ParseResult ( * parts ) <EOL> try : <EOL> return urlparse ( url ) <EOL> except ValueError : <EOL> msg = u'<STR_LIT>' . format ( url ) <EOL> log . debug ( msg , exc_info = True ) <EOL> return None <EOL> def _serp_query_string ( parse_result ) : <EOL> """<STR_LIT>""" <EOL> query = parse_result . query <EOL> if parse_result . fragment != '<STR_LIT>' : <EOL> query = u'<STR_LIT>' . format ( query , parse_result . fragment ) <EOL> return query <EOL> def _is_url_without_path_query_or_fragment ( url_parts ) : <EOL> """<STR_LIT>""" <EOL> return url_parts . path . strip ( '<STR_LIT:/>' ) == '<STR_LIT>' and url_parts . query == '<STR_LIT>' and url_parts . fragment == '<STR_LIT>' <EOL> _engines = None <EOL> def _get_search_engines ( ) : <EOL> """<STR_LIT>""" <EOL> global _engines <EOL> if _engines : <EOL> return _engines <EOL> piwik_engines = _get_piwik_engines ( ) <EOL> get_engine_name = lambda x : x [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] <EOL> definitions_by_engine = groupby ( iteritems ( piwik_engines ) , get_engine_name ) <EOL> _engines = { } <EOL> for engine_name , rule_group in definitions_by_engine : <EOL> defaults = { <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : [ '<STR_LIT:utf-8>' ] <EOL> } <EOL> for i , rule in enumerate ( rule_group ) : <EOL> domain = rule [ <NUM_LIT:0> ] <EOL> rule = rule [ <NUM_LIT:1> ] [ <NUM_LIT:1> : ] <EOL> if i == <NUM_LIT:0> : <EOL> defaults [ '<STR_LIT>' ] = rule [ <NUM_LIT:0> ] <EOL> if len ( rule ) >= <NUM_LIT:2> : <EOL> defaults [ '<STR_LIT>' ] = rule [ <NUM_LIT:1> ] <EOL> if len ( rule ) >= <NUM_LIT:3> : <EOL> defaults [ '<STR_LIT>' ] = rule [ <NUM_LIT:2> ] <EOL> _engines [ domain ] = SearchEngineParser ( engine_name , <EOL> defaults [ '<STR_LIT>' ] , <EOL> defaults [ '<STR_LIT>' ] , <EOL> defaults [ '<STR_LIT>' ] ) <EOL> continue <EOL> args = [ engine_name , defaults [ '<STR_LIT>' ] , <EOL> defaults [ '<STR_LIT>' ] , defaults [ '<STR_LIT>' ] ] <EOL> if len ( rule ) >= <NUM_LIT:1> and rule [ <NUM_LIT:0> ] : <EOL> args [ <NUM_LIT:1> ] = rule [ <NUM_LIT:0> ] <EOL> if len ( rule ) >= <NUM_LIT:2> and rule [ <NUM_LIT:1> ] : <EOL> args [ <NUM_LIT:2> ] = rule [ <NUM_LIT:1> ] <EOL> if len ( rule ) == <NUM_LIT:3> and rule [ <NUM_LIT:2> ] : <EOL> args [ <NUM_LIT:3> ] = rule [ <NUM_LIT:2> ] <EOL> _engines [ domain ] = SearchEngineParser ( * args ) <EOL> return _engines <EOL> def _get_piwik_engines ( ) : <EOL> """<STR_LIT>""" <EOL> stream = pkg_resources . resource_stream <EOL> pickle_path = '<STR_LIT>' . format ( sys . version_info [ <NUM_LIT:0> ] ) <EOL> with stream ( __name__ , pickle_path ) as picklestream : <EOL> _piwik_engines = pickle . load ( picklestream ) <EOL> return _piwik_engines <EOL> _get_lossy_domain_regex = None <EOL> def _get_lossy_domain ( domain ) : <EOL> """<STR_LIT>""" <EOL> global _domain_cache , _get_lossy_domain_regex <EOL> if domain in _domain_cache : <EOL> return _domain_cache [ domain ] <EOL> if not _get_lossy_domain_regex : <EOL> codes = '<STR_LIT:|>' . join ( _country_codes ) <EOL> _get_lossy_domain_regex = re . compile ( <EOL> r'<STR_LIT>' <EOL> r'<STR_LIT>' + <EOL> r'<STR_LIT>' . format ( codes ) + <EOL> r'<STR_LIT>' + <EOL> r'<STR_LIT>' + <EOL> r'<STR_LIT>' . format ( codes ) + <EOL> r'<STR_LIT:$>' ) <EOL> res = _get_lossy_domain_regex . match ( domain ) . groupdict ( ) <EOL> output = u'<STR_LIT>' % ( '<STR_LIT>' if res [ '<STR_LIT>' ] else '<STR_LIT>' , <EOL> res [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' if res [ '<STR_LIT>' ] else res [ '<STR_LIT>' ] or '<STR_LIT>' ) <EOL> _domain_cache [ domain ] = output <EOL> return output <EOL> class ExtractResult ( object ) : <EOL> __slots__ = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def __init__ ( self , engine_name , keyword , parser ) : <EOL> self . engine_name = engine_name <EOL> self . keyword = keyword <EOL> self . parser = parser <EOL> def __repr__ ( self ) : <EOL> repr_fmt = '<STR_LIT>' <EOL> return repr_fmt . format ( self . engine_name , self . keyword , self . parser ) <EOL> class SearchEngineParser ( object ) : <EOL> """<STR_LIT>""" <EOL> __slots__ = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def __init__ ( self , engine_name , keyword_extractor , link_macro , charsets ) : <EOL> """<STR_LIT>""" <EOL> self . engine_name = engine_name <EOL> if isinstance ( keyword_extractor , string_types ) : <EOL> keyword_extractor = [ keyword_extractor ] <EOL> self . keyword_extractor = keyword_extractor [ : ] <EOL> for i , extractor in enumerate ( self . keyword_extractor ) : <EOL> if extractor . startswith ( '<STR_LIT:/>' ) : <EOL> extractor = extractor . strip ( '<STR_LIT:/>' ) <EOL> extractor = re . compile ( extractor ) <EOL> self . keyword_extractor [ i ] = extractor <EOL> self . link_macro = link_macro <EOL> if isinstance ( charsets , string_types ) : <EOL> charsets = [ charsets ] <EOL> self . charsets = [ c . lower ( ) for c in charsets ] <EOL> def get_serp_url ( self , base_url , keyword ) : <EOL> """<STR_LIT>""" <EOL> if self . link_macro is None : <EOL> return None <EOL> link = u'<STR_LIT>' . format ( base_url , self . link_macro . format ( k = keyword ) ) <EOL> return link <EOL> def parse ( self , url_parts ) : <EOL> """<STR_LIT>""" <EOL> original_query = _serp_query_string ( url_parts ) <EOL> query = _unicode_parse_qs ( original_query , keep_blank_values = True ) <EOL> keyword = None <EOL> engine_name = self . engine_name <EOL> if engine_name == '<STR_LIT>' or ( engine_name == '<STR_LIT>' and '<STR_LIT>' in original_query ) : <EOL> engine_name = '<STR_LIT>' <EOL> if '<STR_LIT>' in query : <EOL> prev_query = _unicode_parse_qs ( urlparse ( query [ '<STR_LIT>' ] [ <NUM_LIT:0> ] ) . query ) <EOL> keyword = prev_query . get ( '<STR_LIT:q>' , [ None ] ) [ <NUM_LIT:0> ] <EOL> elif engine_name == '<STR_LIT>' and '<STR_LIT>' in original_query : <EOL> keys = [ ] <EOL> key = query . get ( '<STR_LIT>' ) <EOL> if key : <EOL> keys . append ( key [ <NUM_LIT:0> ] ) <EOL> key = query . get ( '<STR_LIT>' ) <EOL> if key : <EOL> key = key [ <NUM_LIT:0> ] . replace ( '<STR_LIT:+>' , '<STR_LIT>' ) <EOL> keys . append ( key ) <EOL> key = query . get ( '<STR_LIT>' ) <EOL> if key : <EOL> keys . append ( u'<STR_LIT>' . format ( key [ <NUM_LIT:0> ] ) ) <EOL> key = query . get ( '<STR_LIT>' ) <EOL> if key : <EOL> keys . append ( u'<STR_LIT>' . format ( key [ <NUM_LIT:0> ] ) ) <EOL> keyword = u'<STR_LIT:U+0020>' . join ( keys ) . strip ( ) <EOL> if engine_name == '<STR_LIT>' : <EOL> tbm = query . get ( '<STR_LIT>' , [ None ] ) [ <NUM_LIT:0> ] <EOL> if tbm == '<STR_LIT>' : <EOL> engine_name = '<STR_LIT>' <EOL> elif tbm == '<STR_LIT>' : <EOL> engine_name = '<STR_LIT>' <EOL> elif tbm == '<STR_LIT>' : <EOL> engine_name = '<STR_LIT>' <EOL> if keyword is not None : <EOL> return ExtractResult ( engine_name , keyword , self ) <EOL> for extractor in self . keyword_extractor : <EOL> if not isinstance ( extractor , string_types ) : <EOL> match = extractor . search ( url_parts . path ) <EOL> if match : <EOL> keyword = match . group ( <NUM_LIT:1> ) <EOL> break <EOL> else : <EOL> if extractor in query : <EOL> keyword = query [ extractor ] [ - <NUM_LIT:1> ] <EOL> if keyword is None and extractor == '<STR_LIT:q>' and engine_name in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> keyword = '<STR_LIT>' <EOL> elif keyword is None and extractor == '<STR_LIT:q>' and engine_name == '<STR_LIT>' and _is_url_without_path_query_or_fragment ( url_parts ) : <EOL> keyword = '<STR_LIT>' <EOL> elif keyword is None and engine_name == '<STR_LIT>' and url_parts . netloc . lower ( ) == '<STR_LIT>' : <EOL> keyword = '<STR_LIT>' <EOL> if keyword is not None : <EOL> return ExtractResult ( engine_name , keyword , self ) <EOL> def __repr__ ( self ) : <EOL> repr_fmt = ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> return repr_fmt . format ( <EOL> self . engine_name , <EOL> self . keyword_extractor , <EOL> self . link_macro , <EOL> self . charsets ) <EOL> def add_custom_parser ( match_rule , parser ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( match_rule , text_type ) <EOL> assert isinstance ( parser , SearchEngineParser ) <EOL> global _engines <EOL> _get_search_engines ( ) <EOL> _engines [ match_rule ] = parser <EOL> def get_all_query_params ( ) : <EOL> """<STR_LIT>""" <EOL> engines = _get_search_engines ( ) <EOL> all_params = set ( ) <EOL> _not_regex = lambda x : isinstance ( x , string_types ) <EOL> for parser in itervalues ( engines ) : <EOL> params = set ( filter ( _not_regex , parser . keyword_extractor ) ) <EOL> all_params |= params <EOL> return list ( all_params ) <EOL> def get_parser ( referring_url ) : <EOL> """<STR_LIT>""" <EOL> engines = _get_search_engines ( ) <EOL> url_parts = _unicode_urlparse ( referring_url ) <EOL> if url_parts is None : <EOL> return None <EOL> query = _serp_query_string ( url_parts ) <EOL> domain = url_parts . netloc <EOL> path = url_parts . path <EOL> lossy_domain = _get_lossy_domain ( url_parts . netloc ) <EOL> engine_key = url_parts . netloc <EOL> if u'<STR_LIT>' . format ( domain , path ) in engines : <EOL> engine_key = u'<STR_LIT>' . format ( domain , path ) <EOL> elif u'<STR_LIT>' . format ( lossy_domain , path ) in engines : <EOL> engine_key = u'<STR_LIT>' . format ( lossy_domain , path ) <EOL> elif lossy_domain in engines : <EOL> engine_key = lossy_domain <EOL> elif domain not in engines : <EOL> if query [ : <NUM_LIT> ] == '<STR_LIT>' : <EOL> engine_key = '<STR_LIT>' <EOL> elif url_parts . path [ : <NUM_LIT> ] == '<STR_LIT>' : <EOL> engine_key = '<STR_LIT>' <EOL> elif '<STR_LIT>' in url_parts . netloc : <EOL> engine_key = '<STR_LIT>' <EOL> elif '<STR_LIT>' in url_parts . netloc : <EOL> engine_key = '<STR_LIT>' <EOL> else : <EOL> return None <EOL> return engines . get ( engine_key ) <EOL> def is_serp ( referring_url , parser = None , use_naive_method = False ) : <EOL> """<STR_LIT>""" <EOL> res = extract ( referring_url , parser = parser , <EOL> use_naive_method = use_naive_method ) <EOL> return res is not None <EOL> def extract ( serp_url , parser = None , lower_case = True , trimmed = True , <EOL> collapse_whitespace = True , use_naive_method = False ) : <EOL> """<STR_LIT>""" <EOL> url_parts = _unicode_urlparse ( serp_url ) <EOL> if url_parts is None : <EOL> return None <EOL> result = None <EOL> if parser is None : <EOL> parser = get_parser ( url_parts ) <EOL> if parser is None : <EOL> if not use_naive_method : <EOL> return None <EOL> if _naive_re . search ( url_parts . netloc ) : <EOL> query = _unicode_parse_qs ( url_parts . query , keep_blank_values = True ) <EOL> for param in _naive_params : <EOL> if param in query : <EOL> import tldextract <EOL> tld_res = tldextract . extract ( url_parts . netloc ) <EOL> return ExtractResult ( tld_res . domain , <EOL> query [ param ] [ <NUM_LIT:0> ] , <EOL> None ) <EOL> return None <EOL> result = parser . parse ( url_parts ) <EOL> if result is None : <EOL> return None <EOL> if lower_case : <EOL> result . keyword = result . keyword . lower ( ) <EOL> if trimmed : <EOL> result . keyword = result . keyword . strip ( ) <EOL> if collapse_whitespace : <EOL> result . keyword = re . sub ( r'<STR_LIT>' , '<STR_LIT:U+0020>' , result . keyword , re . UNICODE ) <EOL> return result <EOL> def main ( ) : <EOL> import argparse <EOL> import sys <EOL> import re <EOL> parser = argparse . ArgumentParser ( <EOL> description = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT:input>' , metavar = '<STR_LIT:url>' , type = text_type , nargs = '<STR_LIT:*>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT>' , default = False , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> if args . list : <EOL> engines = _get_search_engines ( ) <EOL> engines = sorted ( iteritems ( engines ) , key = lambda x : x [ <NUM_LIT:1> ] . engine_name ) <EOL> print ( '<STR_LIT>' . format ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> for fuzzy_domain , parser in engines : <EOL> print ( '<STR_LIT>' . format ( fuzzy_domain , parser ) ) <EOL> print ( '<STR_LIT>' . format ( len ( engines ) ) ) <EOL> sys . exit ( <NUM_LIT:0> ) <EOL> if len ( args . input ) == <NUM_LIT:0> : <EOL> parser . print_usage ( ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> escape_quotes = lambda s : re . sub ( r'<STR_LIT:">' , '<STR_LIT>' , s ) <EOL> for url in args . input : <EOL> res = extract ( url ) <EOL> if res is None : <EOL> res = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> else : <EOL> res = [ escape_quotes ( res . engine_name ) , escape_quotes ( res . keyword ) ] <EOL> res = [ u'<STR_LIT>' . format ( r ) for r in res ] <EOL> print ( u'<STR_LIT:U+002C>' . join ( res ) ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
