<s> from __future__ import absolute_import <EOL> import cgi <EOL> import collections <EOL> import datetime <EOL> import functools <EOL> import io <EOL> import logging <EOL> import os . path <EOL> import random <EOL> import time <EOL> import requests . exceptions <EOL> import six <EOL> import six . moves . urllib_parse as urlparse <EOL> from requestbuilder . exceptions import ( ClientError , ServerError , <EOL> ServiceInitError , TimeoutError ) <EOL> from requestbuilder . mixins import RegionConfigurableMixin <EOL> class BaseService ( RegionConfigurableMixin ) : <EOL> NAME = None <EOL> DESCRIPTION = '<STR_LIT>' <EOL> API_VERSION = '<STR_LIT>' <EOL> MAX_RETRIES = <NUM_LIT:2> <EOL> TIMEOUT = <NUM_LIT:30> <EOL> REGION_ENVVAR = None <EOL> URL_ENVVAR = None <EOL> ARGS = [ ] <EOL> def __init__ ( self , config , loglevel = None , max_retries = None , timeout = None , <EOL> ** kwargs ) : <EOL> self . args = kwargs <EOL> self . config = config <EOL> self . endpoint = None <EOL> self . log = logging . getLogger ( self . __class__ . __name__ ) <EOL> if loglevel is not None : <EOL> self . log . level = loglevel <EOL> self . max_retries = max_retries <EOL> self . region_name = None <EOL> self . session_args = { } <EOL> self . timeout = timeout <EOL> self . _session = None <EOL> @ classmethod <EOL> def from_other ( cls , other , ** kwargs ) : <EOL> kwargs . setdefault ( '<STR_LIT>' , other . log . level ) <EOL> kwargs . setdefault ( '<STR_LIT>' , other . max_retries ) <EOL> kwargs . setdefault ( '<STR_LIT>' , dict ( other . session_args ) ) <EOL> kwargs . setdefault ( '<STR_LIT>' , other . timeout ) <EOL> if '<STR_LIT>' in other . args : <EOL> kwargs . setdefault ( '<STR_LIT>' , other . args [ '<STR_LIT>' ] ) <EOL> new = cls ( other . config , ** kwargs ) <EOL> new . configure ( ) <EOL> return new <EOL> def configure ( self ) : <EOL> self . update_config_view ( ) <EOL> self . __configure_endpoint ( ) <EOL> if self . max_retries is None : <EOL> config_max_retries = self . config . get_global_option ( '<STR_LIT>' ) <EOL> if config_max_retries is not None : <EOL> self . max_retries = int ( config_max_retries ) <EOL> else : <EOL> self . max_retries = self . MAX_RETRIES <EOL> if self . timeout is None : <EOL> config_timeout = self . config . get_global_option ( '<STR_LIT>' ) <EOL> if config_timeout is not None : <EOL> self . timeout = float ( config_timeout ) <EOL> else : <EOL> self . timeout = self . TIMEOUT <EOL> self . session_args . setdefault ( '<STR_LIT>' , True ) <EOL> self . session_args . setdefault ( '<STR_LIT>' , self . config . convert_to_bool ( <EOL> self . config . get_region_option ( '<STR_LIT>' ) , default = False ) ) <EOL> self . validate_config ( ) <EOL> @ property <EOL> def session ( self ) : <EOL> if self . _session is None : <EOL> self . _session = requests . session ( ) <EOL> for key , val in self . session_args . iteritems ( ) : <EOL> setattr ( self . _session , key , val ) <EOL> for adapter in self . _session . adapters . values ( ) : <EOL> adapter . max_retries = <NUM_LIT:0> <EOL> return self . _session <EOL> def validate_config ( self ) : <EOL> if self . endpoint is None : <EOL> if self . NAME is not None : <EOL> url_opt = '<STR_LIT>' . format ( self . NAME ) <EOL> available_regions = self . config . get_all_region_options ( url_opt ) <EOL> if len ( available_regions ) > <NUM_LIT:0> : <EOL> msg = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) . format ( self . NAME , <EOL> '<STR_LIT:U+002CU+0020>' . join ( sorted ( available_regions ) ) ) <EOL> else : <EOL> msg = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) . format ( self . NAME , url_opt ) <EOL> else : <EOL> msg = '<STR_LIT>' <EOL> raise ServiceInitError ( msg ) <EOL> def get_request_url ( self , method = '<STR_LIT:GET>' , path = None , params = None , <EOL> headers = None , data = None , files = None , auth = None ) : <EOL> url = self . __get_url_for_path ( path ) <EOL> headers = dict ( headers or { } ) <EOL> if '<STR_LIT:host>' not in [ header . lower ( ) for header in headers ] : <EOL> headers [ '<STR_LIT>' ] = urlparse . urlparse ( self . endpoint ) . netloc <EOL> p_request = self . __log_and_prepare_request ( method , url , params , data , <EOL> files , headers , auth ) <EOL> return p_request . url <EOL> def send_request ( self , method = '<STR_LIT:GET>' , path = None , params = None , headers = None , <EOL> data = None , files = None , auth = None ) : <EOL> url = self . __get_url_for_path ( path ) <EOL> headers = dict ( headers ) <EOL> if '<STR_LIT:host>' not in [ header . lower ( ) for header in headers ] : <EOL> headers [ '<STR_LIT>' ] = urlparse . urlparse ( self . endpoint ) . netloc <EOL> try : <EOL> max_tries = self . max_retries + <NUM_LIT:1> <EOL> assert max_tries >= <NUM_LIT:1> <EOL> redirects_left = <NUM_LIT:5> <EOL> if isinstance ( data , file ) and hasattr ( data , '<STR_LIT>' ) : <EOL> data_file_offset = data . tell ( ) <EOL> else : <EOL> data_file_offset = None <EOL> while True : <EOL> for attempt_no , delay in enumerate ( <EOL> _generate_delays ( max_tries ) , <NUM_LIT:1> ) : <EOL> if delay > <NUM_LIT:0> : <EOL> self . log . debug ( '<STR_LIT>' , delay ) <EOL> time . sleep ( delay ) <EOL> self . log . info ( '<STR_LIT>' , <EOL> attempt_no , max_tries ) <EOL> p_request = self . __log_and_prepare_request ( <EOL> method , url , params , data , files , headers , auth ) <EOL> p_request . start_time = datetime . datetime . now ( ) <EOL> proxies = requests . utils . get_environ_proxies ( url ) <EOL> for key , val in sorted ( proxies . items ( ) ) : <EOL> self . log . debug ( '<STR_LIT>' , key , val ) <EOL> try : <EOL> response = self . session . send ( <EOL> p_request , timeout = self . timeout , proxies = proxies , <EOL> allow_redirects = False ) <EOL> except requests . exceptions . Timeout : <EOL> if attempt_no < max_tries : <EOL> self . log . debug ( '<STR_LIT>' , exc_info = True ) <EOL> if data_file_offset is not None : <EOL> self . log . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> data . seek ( data_file_offset ) <EOL> continue <EOL> elif not hasattr ( data , '<STR_LIT>' ) : <EOL> continue <EOL> raise <EOL> if response . status_code not in ( <NUM_LIT> , <NUM_LIT> ) : <EOL> break <EOL> if ( response . status_code in ( <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ) and <EOL> redirects_left > <NUM_LIT:0> and '<STR_LIT>' in response . headers ) : <EOL> redirects_left -= <NUM_LIT:1> <EOL> parsed_rdr = urlparse . urlparse ( <EOL> response . headers [ '<STR_LIT>' ] ) <EOL> parsed_url = urlparse . urlparse ( url ) <EOL> new_url_bits = [ ] <EOL> for rdr_bit , url_bit in zip ( parsed_rdr , parsed_url ) : <EOL> new_url_bits . append ( rdr_bit or url_bit ) <EOL> if '<STR_LIT>' in headers : <EOL> headers [ '<STR_LIT>' ] = new_url_bits [ <NUM_LIT:1> ] <EOL> url = urlparse . urlunparse ( new_url_bits ) <EOL> self . log . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , url , redirects_left ) <EOL> if data_file_offset is not None : <EOL> self . log . debug ( '<STR_LIT>' ) <EOL> data . seek ( data_file_offset ) <EOL> continue <EOL> elif response . status_code >= <NUM_LIT> : <EOL> self . handle_http_error ( response ) <EOL> return response <EOL> except requests . exceptions . Timeout as exc : <EOL> self . log . debug ( '<STR_LIT>' , exc_info = True ) <EOL> raise TimeoutError ( '<STR_LIT>' , exc ) <EOL> except requests . exceptions . ConnectionError as exc : <EOL> self . log . debug ( '<STR_LIT>' , exc_info = True ) <EOL> return self . __handle_connection_error ( exc ) <EOL> except requests . exceptions . HTTPError as exc : <EOL> return self . handle_http_error ( response ) <EOL> except requests . exceptions . RequestException as exc : <EOL> self . log . debug ( '<STR_LIT>' , exc_info = True ) <EOL> raise ClientError ( exc ) <EOL> def __handle_connection_error ( self , err ) : <EOL> if isinstance ( err , six . string_types ) : <EOL> msg = err <EOL> elif isinstance ( err , Exception ) and len ( err . args ) > <NUM_LIT:0> : <EOL> if hasattr ( err . args [ <NUM_LIT:0> ] , '<STR_LIT>' ) : <EOL> msg = err . args [ <NUM_LIT:0> ] . reason <EOL> elif isinstance ( err . args [ <NUM_LIT:0> ] , Exception ) : <EOL> return self . __handle_connection_error ( err . args [ <NUM_LIT:0> ] ) <EOL> else : <EOL> msg = err . args [ <NUM_LIT:0> ] <EOL> else : <EOL> raise ClientError ( '<STR_LIT>' ) <EOL> raise ClientError ( '<STR_LIT>' . format ( msg ) ) <EOL> def handle_http_error ( self , response ) : <EOL> self . log . debug ( '<STR_LIT>' , exc_info = True ) <EOL> raise ServerError ( response ) <EOL> def __get_url_for_path ( self , path ) : <EOL> if path : <EOL> if self . endpoint . endswith ( '<STR_LIT:/>' ) : <EOL> return self . endpoint + path <EOL> else : <EOL> return self . endpoint + '<STR_LIT:/>' + path <EOL> else : <EOL> return self . endpoint <EOL> def __log_and_prepare_request ( self , method , url , params , data , files , <EOL> headers , auth ) : <EOL> hooks = { '<STR_LIT>' : functools . partial ( _log_response_data , self . log ) } <EOL> if auth : <EOL> bound_auth = auth . bind_to_service ( self ) <EOL> else : <EOL> bound_auth = None <EOL> request = requests . Request ( method = method , url = url , params = params , <EOL> data = data , files = files , headers = headers , <EOL> auth = bound_auth ) <EOL> p_request = self . session . prepare_request ( request ) <EOL> p_request . hooks = { '<STR_LIT>' : hooks [ '<STR_LIT>' ] } <EOL> self . log . debug ( '<STR_LIT>' , request . method ) <EOL> self . log . debug ( '<STR_LIT>' , p_request . url ) <EOL> if isinstance ( p_request . headers , ( dict , collections . Mapping ) ) : <EOL> for key , val in sorted ( p_request . headers . iteritems ( ) ) : <EOL> if key . lower ( ) . endswith ( '<STR_LIT:password>' ) : <EOL> val = '<STR_LIT>' <EOL> self . log . debug ( '<STR_LIT>' , key , val ) <EOL> if isinstance ( request . params , ( dict , collections . Mapping ) ) : <EOL> for key , val in sorted ( urlparse . parse_qsl ( <EOL> urlparse . urlparse ( p_request . url ) . query , <EOL> keep_blank_values = True ) ) : <EOL> if key . lower ( ) . endswith ( '<STR_LIT:password>' ) : <EOL> val = '<STR_LIT>' <EOL> self . log . debug ( '<STR_LIT>' , key , val ) <EOL> if isinstance ( request . data , ( dict , collections . Mapping ) ) : <EOL> content_type , content_type_params = cgi . parse_header ( <EOL> p_request . headers . get ( '<STR_LIT>' ) or '<STR_LIT>' ) <EOL> if content_type == '<STR_LIT>' : <EOL> data = cgi . parse_multipart ( io . BytesIO ( p_request . body ) , <EOL> content_type_params ) <EOL> elif content_type == '<STR_LIT>' : <EOL> data = dict ( urlparse . parse_qsl ( p_request . body , <EOL> keep_blank_values = True ) ) <EOL> else : <EOL> data = request . data <EOL> for key , val in sorted ( data . items ( ) ) : <EOL> if key in ( request . files or { } ) : <EOL> continue <EOL> if key . lower ( ) . endswith ( '<STR_LIT:password>' ) : <EOL> val = '<STR_LIT>' <EOL> self . log . debug ( '<STR_LIT>' , key , val ) <EOL> if isinstance ( request . files , ( dict , collections . Mapping ) ) : <EOL> for key , val in sorted ( request . files . iteritems ( ) ) : <EOL> if hasattr ( val , '<STR_LIT>' ) : <EOL> val = '<STR_LIT>' . format ( len ( val ) ) <EOL> self . log . debug ( '<STR_LIT>' , key , val ) <EOL> return p_request <EOL> def __configure_endpoint ( self ) : <EOL> if self . args . get ( '<STR_LIT:url>' ) : <EOL> url , region_name = _parse_endpoint_url ( self . args [ '<STR_LIT:url>' ] ) <EOL> elif os . getenv ( self . URL_ENVVAR ) : <EOL> url , region_name = _parse_endpoint_url ( os . getenv ( self . URL_ENVVAR ) ) <EOL> elif self . NAME : <EOL> url , section = self . config . get_region_option2 ( self . NAME + '<STR_LIT>' ) <EOL> if section : <EOL> region_name = self . config . get_region_option ( '<STR_LIT:name>' , section ) <EOL> if region_name is None : <EOL> region_name = section . rsplit ( '<STR_LIT::>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> else : <EOL> region_name = None <EOL> self . endpoint = url <EOL> self . region_name = region_name <EOL> def _log_response_data ( logger , response , ** _ ) : <EOL> if hasattr ( response . request , '<STR_LIT>' ) : <EOL> duration = datetime . datetime . now ( ) - response . request . start_time <EOL> logger . debug ( '<STR_LIT>' , duration . seconds , <EOL> duration . microseconds // <NUM_LIT:1000> ) <EOL> if response . status_code >= <NUM_LIT> : <EOL> logger . error ( '<STR_LIT>' , response . status_code ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' , response . status_code ) <EOL> if isinstance ( response . headers , ( dict , collections . Mapping ) ) : <EOL> for key , val in sorted ( response . headers . items ( ) ) : <EOL> logger . debug ( '<STR_LIT>' , key , val ) <EOL> def _generate_delays ( max_tries ) : <EOL> if max_tries >= <NUM_LIT:1> : <EOL> yield <NUM_LIT:0> <EOL> for retry_no in range ( <NUM_LIT:1> , max_tries ) : <EOL> next_delay = ( random . random ( ) + <NUM_LIT:1> ) * <NUM_LIT:2> ** ( retry_no - <NUM_LIT:1> ) <EOL> yield min ( ( next_delay , <NUM_LIT:15> ) ) <EOL> def _parse_endpoint_url ( urlish ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' in urlish : <EOL> region , url = urlish . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> else : <EOL> region = None <EOL> url = urlish <EOL> return url , region </s>
