<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import logging <EOL> import os <EOL> from random import randint <EOL> from copy import copy , deepcopy <EOL> import threading <EOL> import itertools <EOL> import collections <EOL> from six . moves . urllib . parse import urljoin <EOL> import email <EOL> from datetime import datetime <EOL> import weakref <EOL> import six <EOL> from weblib . html import find_base_url <EOL> from weblib . http import normalize_http_values <EOL> from grab . document import Document <EOL> from grab import error <EOL> from grab . cookie import CookieManager <EOL> from grab . proxylist import ProxyList , parse_proxy_line <EOL> from grab . deprecated import DeprecatedThings <EOL> from grab . util . warning import warn <EOL> __all__ = ( '<STR_LIT>' , ) <EOL> REQUEST_COUNTER = itertools . count ( <NUM_LIT:1> ) <EOL> GLOBAL_STATE = { <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> } <EOL> MUTABLE_CONFIG_KEYS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> TRANSPORT_CACHE = { } <EOL> TRANSPORT_ALIAS = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> logger_network = logging . getLogger ( '<STR_LIT>' ) <EOL> def reset_request_counter ( ) : <EOL> global REQUEST_COUNTER <EOL> REQUEST_COUNTER = itertools . count ( <NUM_LIT:1> ) <EOL> def copy_config ( config , mutable_config_keys = MUTABLE_CONFIG_KEYS ) : <EOL> """<STR_LIT>""" <EOL> cloned_config = copy ( config ) <EOL> for key in mutable_config_keys : <EOL> cloned_config [ key ] = copy ( config [ key ] ) <EOL> return cloned_config <EOL> def default_config ( ) : <EOL> return dict ( <EOL> url = None , <EOL> log_file = None , <EOL> log_dir = False , <EOL> debug_post = False , <EOL> debug_post_limit = <NUM_LIT> , <EOL> debug = False , <EOL> verbose_logging = False , <EOL> webdriver = '<STR_LIT>' , <EOL> selenium_wait = <NUM_LIT:1> , <EOL> proxy = None , <EOL> proxy_type = None , <EOL> proxy_userpwd = None , <EOL> proxy_auto_change = True , <EOL> method = None , <EOL> post = None , <EOL> multipart_post = None , <EOL> headers = { } , <EOL> common_headers = { } , <EOL> user_agent = None , <EOL> user_agent_file = None , <EOL> referer = None , <EOL> reuse_referer = True , <EOL> cookies = { } , <EOL> reuse_cookies = True , <EOL> cookiefile = None , <EOL> timeout = <NUM_LIT:15> , <EOL> connect_timeout = <NUM_LIT:3> , <EOL> connection_reuse = True , <EOL> nobody = False , <EOL> body_maxsize = None , <EOL> body_inmemory = True , <EOL> body_storage_dir = None , <EOL> body_storage_filename = None , <EOL> body_storage_create_dir = False , <EOL> reject_file_size = None , <EOL> encoding = '<STR_LIT>' , <EOL> interface = None , <EOL> follow_refresh = False , <EOL> follow_location = True , <EOL> redirect_limit = <NUM_LIT:10> , <EOL> userpwd = None , <EOL> charset = '<STR_LIT:utf-8>' , <EOL> document_charset = None , <EOL> content_type = '<STR_LIT:html>' , <EOL> fix_special_entities = True , <EOL> lowercased_tree = False , <EOL> strip_null_bytes = True , <EOL> state = { } , <EOL> ) <EOL> class Grab ( DeprecatedThings ) : <EOL> __slots__ = ( '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> clonable_attributes = ( '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> mutable_config_keys = copy ( MUTABLE_CONFIG_KEYS ) <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , document_body = None , <EOL> transport = '<STR_LIT>' , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . meta = { } <EOL> self . _doc = None <EOL> self . config = default_config ( ) <EOL> self . config [ '<STR_LIT>' ] = self . common_headers ( ) <EOL> self . cookies = CookieManager ( ) <EOL> self . proxylist = ProxyList ( ) <EOL> self . setup_transport ( transport ) <EOL> self . reset ( ) <EOL> if kwargs : <EOL> self . setup ( ** kwargs ) <EOL> if document_body is not None : <EOL> self . setup_document ( document_body ) <EOL> def _get_doc ( self ) : <EOL> if self . _doc is None : <EOL> self . _doc = Document ( self ) <EOL> return self . _doc <EOL> def _set_doc ( self , obj ) : <EOL> self . _doc = obj <EOL> doc = property ( _get_doc , _set_doc ) <EOL> def setup_transport ( self , transport_param ) : <EOL> self . transport_param = transport_param <EOL> if isinstance ( transport_param , six . string_types ) : <EOL> if transport_param in TRANSPORT_ALIAS : <EOL> transport_param = TRANSPORT_ALIAS [ transport_param ] <EOL> if not '<STR_LIT:.>' in transport_param : <EOL> raise error . GrabMisuseError ( '<STR_LIT>' <EOL> % transport_param ) <EOL> else : <EOL> mod_path , cls_name = transport_param . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) <EOL> try : <EOL> cls = TRANSPORT_CACHE [ ( mod_path , cls_name ) ] <EOL> except KeyError : <EOL> mod = __import__ ( mod_path , globals ( ) , locals ( ) , [ '<STR_LIT:foo>' ] ) <EOL> cls = getattr ( mod , cls_name ) <EOL> TRANSPORT_CACHE [ ( mod_path , cls_name ) ] = cls <EOL> self . transport = cls ( ) <EOL> elif isinstance ( transport_param , collections . Callable ) : <EOL> self . transport = transport_param ( ) <EOL> else : <EOL> raise error . GrabMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % type ( transport_param ) ) <EOL> def reset ( self ) : <EOL> """<STR_LIT>""" <EOL> self . request_head = None <EOL> self . request_body = None <EOL> self . request_method = None <EOL> self . transport . reset ( ) <EOL> def clone ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> g = Grab ( transport = self . transport_param ) <EOL> g . config = self . dump_config ( ) <EOL> g . doc = self . doc . copy ( ) <EOL> g . doc . grab = weakref . proxy ( g ) <EOL> for key in self . clonable_attributes : <EOL> setattr ( g , key , getattr ( self , key ) ) <EOL> g . cookies = deepcopy ( self . cookies ) <EOL> if kwargs : <EOL> g . setup ( ** kwargs ) <EOL> return g <EOL> def adopt ( self , g ) : <EOL> """<STR_LIT>""" <EOL> self . load_config ( g . config ) <EOL> self . doc = g . doc . copy ( new_grab = self ) <EOL> for key in self . clonable_attributes : <EOL> setattr ( self , key , getattr ( g , key ) ) <EOL> self . cookies = deepcopy ( g . cookies ) <EOL> def dump_config ( self ) : <EOL> """<STR_LIT>""" <EOL> conf = copy_config ( self . config , self . mutable_config_keys ) <EOL> conf [ '<STR_LIT:state>' ] = { <EOL> '<STR_LIT>' : list ( self . cookies . cookiejar ) , <EOL> } <EOL> return conf <EOL> def load_config ( self , config ) : <EOL> """<STR_LIT>""" <EOL> self . config = copy_config ( config , self . mutable_config_keys ) <EOL> if '<STR_LIT>' in config [ '<STR_LIT:state>' ] : <EOL> self . cookies = CookieManager . from_cookie_list ( <EOL> config [ '<STR_LIT:state>' ] [ '<STR_LIT>' ] ) <EOL> def setup ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' in kwargs : <EOL> warn ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> del kwargs [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in kwargs : <EOL> warn ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> del kwargs [ '<STR_LIT>' ] <EOL> for key in kwargs : <EOL> if key not in self . config . keys ( ) : <EOL> raise error . GrabMisuseError ( '<STR_LIT>' % key ) <EOL> if '<STR_LIT:url>' in kwargs : <EOL> if self . config . get ( '<STR_LIT:url>' ) : <EOL> kwargs [ '<STR_LIT:url>' ] = self . make_url_absolute ( kwargs [ '<STR_LIT:url>' ] ) <EOL> self . config . update ( kwargs ) <EOL> def go ( self , url , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . request ( url = url , ** kwargs ) <EOL> def download ( self , url , location , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> doc = self . go ( url , ** kwargs ) <EOL> with open ( location , '<STR_LIT:wb>' ) as out : <EOL> out . write ( doc . body ) <EOL> return len ( doc . body ) <EOL> def prepare_request ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . reset ( ) <EOL> self . request_counter = next ( REQUEST_COUNTER ) <EOL> if kwargs : <EOL> self . setup ( ** kwargs ) <EOL> if self . proxylist . size ( ) and self . config [ '<STR_LIT>' ] : <EOL> self . change_proxy ( ) <EOL> self . request_method = self . detect_request_method ( ) <EOL> self . transport . process_config ( self ) <EOL> def log_request ( self , extra = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> thread_name = threading . currentThread ( ) . getName ( ) . lower ( ) <EOL> if thread_name == '<STR_LIT>' : <EOL> thread_name = '<STR_LIT>' <EOL> else : <EOL> thread_name = '<STR_LIT>' % thread_name <EOL> if self . config [ '<STR_LIT>' ] : <EOL> if self . config [ '<STR_LIT>' ] : <EOL> auth = '<STR_LIT>' <EOL> else : <EOL> auth = '<STR_LIT>' <EOL> proxy_info = '<STR_LIT>' % ( <EOL> self . config [ '<STR_LIT>' ] , self . config [ '<STR_LIT>' ] , auth ) <EOL> else : <EOL> proxy_info = '<STR_LIT>' <EOL> if extra : <EOL> extra = '<STR_LIT>' % extra <EOL> logger_network . debug ( '<STR_LIT>' , <EOL> self . request_counter , thread_name , <EOL> extra , self . request_method or '<STR_LIT:GET>' , <EOL> self . config [ '<STR_LIT:url>' ] , proxy_info ) <EOL> def request ( self , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . prepare_request ( ** kwargs ) <EOL> refresh_count = <NUM_LIT:0> <EOL> while True : <EOL> self . log_request ( ) <EOL> try : <EOL> self . transport . request ( ) <EOL> except error . GrabError : <EOL> self . reset_temporary_options ( ) <EOL> self . save_failed_dump ( ) <EOL> raise <EOL> else : <EOL> doc = self . process_request_result ( ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> if doc . code in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) : <EOL> if doc . headers . get ( '<STR_LIT>' ) : <EOL> refresh_count += <NUM_LIT:1> <EOL> if refresh_count > self . config [ '<STR_LIT>' ] : <EOL> raise error . GrabTooManyRedirectsError ( ) <EOL> else : <EOL> url = doc . headers . get ( '<STR_LIT>' ) <EOL> self . prepare_request ( <EOL> url = self . make_url_absolute ( url ) , <EOL> referer = None ) <EOL> continue <EOL> if self . config [ '<STR_LIT>' ] : <EOL> refresh_url = self . doc . get_meta_refresh_url ( ) <EOL> if refresh_url is not None : <EOL> refresh_count += <NUM_LIT:1> <EOL> if refresh_count > self . config [ '<STR_LIT>' ] : <EOL> raise error . GrabTooManyRedirectsError ( ) <EOL> else : <EOL> self . prepare_request ( <EOL> url = self . make_url_absolute ( refresh_url ) , <EOL> referer = None ) <EOL> continue <EOL> return doc <EOL> def process_request_result ( self , prepare_response_func = None ) : <EOL> """<STR_LIT>""" <EOL> now = datetime . utcnow ( ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> post = self . config [ '<STR_LIT>' ] or self . config [ '<STR_LIT>' ] <EOL> if isinstance ( post , dict ) : <EOL> post = list ( post . items ( ) ) <EOL> if post : <EOL> if isinstance ( post , six . string_types ) : <EOL> post = post [ : self . config [ '<STR_LIT>' ] ] + '<STR_LIT>' <EOL> else : <EOL> items = normalize_http_values ( post , charset = '<STR_LIT:utf-8>' ) <EOL> new_items = [ ] <EOL> for key , value in items : <EOL> if len ( value ) > self . config [ '<STR_LIT>' ] : <EOL> value = value [ <EOL> : self . config [ '<STR_LIT>' ] ] + '<STR_LIT>' <EOL> else : <EOL> value = value <EOL> new_items . append ( ( key , value ) ) <EOL> post = '<STR_LIT:\n>' . join ( '<STR_LIT>' % x for x in new_items ) <EOL> if post : <EOL> logger_network . debug ( '<STR_LIT>' <EOL> % ( self . request_counter , post ) ) <EOL> self . reset_temporary_options ( ) <EOL> if prepare_response_func : <EOL> self . doc = prepare_response_func ( self . transport , self ) <EOL> else : <EOL> self . doc = self . transport . prepare_response ( self ) <EOL> if self . doc . grab is None : <EOL> self . doc . grab = weakref . proxy ( self ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> self . cookies . update ( self . doc . cookies ) <EOL> self . doc . timestamp = now <EOL> self . config [ '<STR_LIT>' ] = self . doc . charset <EOL> if self . config [ '<STR_LIT>' ] : <EOL> with open ( self . config [ '<STR_LIT>' ] , '<STR_LIT:wb>' ) as out : <EOL> out . write ( self . doc . body ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> self . cookies . save_to_file ( self . config [ '<STR_LIT>' ] ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> self . config [ '<STR_LIT>' ] = self . doc . url <EOL> self . copy_request_data ( ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> self . save_dumps ( ) <EOL> return self . doc <EOL> def reset_temporary_options ( self ) : <EOL> self . config [ '<STR_LIT>' ] = None <EOL> self . config [ '<STR_LIT>' ] = None <EOL> self . config [ '<STR_LIT>' ] = None <EOL> self . config [ '<STR_LIT>' ] = None <EOL> def save_failed_dump ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> if self . transport_param == '<STR_LIT>' : <EOL> pass <EOL> else : <EOL> self . doc = self . transport . prepare_response ( self ) <EOL> self . copy_request_data ( ) <EOL> if self . config [ '<STR_LIT>' ] : <EOL> self . save_dumps ( ) <EOL> except Exception as ex : <EOL> logger . error ( '<STR_LIT>' , exc_info = ex ) <EOL> def copy_request_data ( self ) : <EOL> self . request_head = self . transport . request_head <EOL> self . request_body = self . transport . request_body <EOL> def setup_document ( self , content , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . reset ( ) <EOL> if isinstance ( content , six . text_type ) : <EOL> raise error . GrabMisuseError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> doc = Document ( grab = self ) <EOL> doc . body = content <EOL> doc . status = '<STR_LIT>' <EOL> doc . head = b'<STR_LIT>' <EOL> doc . parse ( charset = kwargs . get ( '<STR_LIT>' ) ) <EOL> doc . code = <NUM_LIT:200> <EOL> doc . total_time = <NUM_LIT:0> <EOL> doc . connect_time = <NUM_LIT:0> <EOL> doc . name_lookup_time = <NUM_LIT:0> <EOL> doc . url = '<STR_LIT>' <EOL> for key , value in kwargs . items ( ) : <EOL> setattr ( doc , key , value ) <EOL> self . doc = doc <EOL> def change_proxy ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . proxylist . size ( ) : <EOL> proxy = self . proxylist . get_random_proxy ( ) <EOL> self . setup ( proxy = proxy . get_address ( ) , <EOL> proxy_userpwd = proxy . get_userpwd ( ) , <EOL> proxy_type = proxy . proxy_type ) <EOL> else : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> def use_next_proxy ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . proxylist . size ( ) : <EOL> proxy = self . proxylist . get_next_proxy ( ) <EOL> self . setup ( proxy = proxy . get_address ( ) , <EOL> proxy_userpwd = proxy . get_userpwd ( ) , <EOL> proxy_type = proxy . proxy_type ) <EOL> else : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> """<STR_LIT>""" <EOL> def common_headers ( self ) : <EOL> """<STR_LIT>""" <EOL> return { <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % randint ( <NUM_LIT:2> , <NUM_LIT:5> ) , <EOL> '<STR_LIT>' : '<STR_LIT>' % ( randint ( <NUM_LIT:5> , <NUM_LIT:9> ) ) , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> % randint ( <NUM_LIT:5> , <NUM_LIT:7> ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> def save_dumps ( self ) : <EOL> thread_name = threading . currentThread ( ) . getName ( ) . lower ( ) <EOL> if thread_name == '<STR_LIT>' : <EOL> thread_name = '<STR_LIT>' <EOL> else : <EOL> thread_name = '<STR_LIT>' % thread_name <EOL> file_name = os . path . join ( self . config [ '<STR_LIT>' ] , '<STR_LIT>' % ( <EOL> self . request_counter , thread_name ) ) <EOL> with open ( file_name , '<STR_LIT:wb>' ) as out : <EOL> out . write ( b'<STR_LIT>' ) <EOL> out . write ( self . request_head ) <EOL> out . write ( b'<STR_LIT:\n>' ) <EOL> out . write ( b'<STR_LIT>' ) <EOL> out . write ( self . request_body ) <EOL> out . write ( b'<STR_LIT>' ) <EOL> out . write ( b'<STR_LIT>' ) <EOL> out . write ( self . doc . head ) <EOL> file_extension = '<STR_LIT:html>' <EOL> file_name = os . path . join ( self . config [ '<STR_LIT>' ] , '<STR_LIT>' % ( <EOL> self . request_counter , thread_name , file_extension ) ) <EOL> self . doc . save ( file_name ) <EOL> def make_url_absolute ( self , url , resolve_base = False ) : <EOL> """<STR_LIT>""" <EOL> if self . config [ '<STR_LIT:url>' ] : <EOL> if resolve_base : <EOL> ubody = self . doc . unicode_body ( ) <EOL> base_url = find_base_url ( ubody ) <EOL> if base_url : <EOL> return urljoin ( base_url , url ) <EOL> return urljoin ( self . config [ '<STR_LIT:url>' ] , url ) <EOL> else : <EOL> return url <EOL> def detect_request_method ( self ) : <EOL> """<STR_LIT>""" <EOL> method = self . config [ '<STR_LIT>' ] <EOL> if method : <EOL> method = method . upper ( ) <EOL> else : <EOL> if self . config [ '<STR_LIT>' ] or self . config [ '<STR_LIT>' ] : <EOL> method = '<STR_LIT:POST>' <EOL> else : <EOL> method = '<STR_LIT:GET>' <EOL> return method <EOL> def clear_cookies ( self ) : <EOL> """<STR_LIT>""" <EOL> self . config [ '<STR_LIT>' ] = { } <EOL> self . cookies . clear ( ) <EOL> def setup_with_proxyline ( self , line , proxy_type = '<STR_LIT:http>' ) : <EOL> host , port , user , pwd = parse_proxy_line ( line ) <EOL> server_port = '<STR_LIT>' % ( host , port ) <EOL> self . setup ( proxy = server_port , proxy_type = proxy_type ) <EOL> if user : <EOL> userpwd = '<STR_LIT>' % ( user , pwd ) <EOL> self . setup ( proxy_userpwd = userpwd ) <EOL> def __getstate__ ( self ) : <EOL> """<STR_LIT>""" <EOL> state = { } <EOL> for cls in type ( self ) . mro ( ) : <EOL> cls_slots = getattr ( cls , '<STR_LIT>' , ( ) ) <EOL> for slot in cls_slots : <EOL> if slot != '<STR_LIT>' : <EOL> if hasattr ( self , slot ) : <EOL> state [ slot ] = getattr ( self , slot ) <EOL> if state [ '<STR_LIT>' ] : <EOL> state [ '<STR_LIT>' ] . grab = weakref . proxy ( self ) <EOL> return state <EOL> def __setstate__ ( self , state ) : <EOL> for slot , value in state . items ( ) : <EOL> setattr ( self , slot , value ) <EOL> @ property <EOL> def request_headers ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> first_head = self . request_head . split ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> lines = first_head . split ( '<STR_LIT:\r\n>' ) <EOL> lines = [ x for x in lines if '<STR_LIT::>' in x ] <EOL> headers = email . message_from_string ( '<STR_LIT:\n>' . join ( lines ) ) <EOL> return headers <EOL> except Exception as ex : <EOL> logger . error ( '<STR_LIT>' , exc_info = ex ) <EOL> return { } <EOL> def dump ( self ) : <EOL> """<STR_LIT>""" <EOL> self . doc . save ( '<STR_LIT>' ) <EOL> BaseGrab = Grab </s>
