<s> """<STR_LIT>""" <EOL> import copy <EOL> import httplib <EOL> import logging <EOL> import random <EOL> import ssl <EOL> import StringIO <EOL> import httparchive <EOL> import platformsettings <EOL> import script_injector <EOL> try : <EOL> import Image <EOL> except ImportError : <EOL> Image = None <EOL> TIMER = platformsettings . timer <EOL> class HttpClientException ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def _InjectScripts ( response , inject_script ) : <EOL> """<STR_LIT>""" <EOL> if type ( response ) == tuple : <EOL> logging . warn ( '<STR_LIT>' , response ) <EOL> content_type = response . get_header ( '<STR_LIT>' ) <EOL> if content_type and content_type . startswith ( '<STR_LIT>' ) : <EOL> text = response . get_data_as_text ( ) <EOL> text , already_injected = script_injector . InjectScript ( <EOL> text , '<STR_LIT>' , inject_script ) <EOL> if not already_injected : <EOL> response = copy . deepcopy ( response ) <EOL> response . set_data ( text ) <EOL> return response <EOL> def _ScrambleImages ( response ) : <EOL> """<STR_LIT>""" <EOL> assert Image , '<STR_LIT>' <EOL> content_type = response . get_header ( '<STR_LIT>' ) <EOL> if content_type and content_type . startswith ( '<STR_LIT>' ) : <EOL> try : <EOL> image_data = response . response_data [ <NUM_LIT:0> ] <EOL> image_data . decode ( encoding = '<STR_LIT>' ) <EOL> im = Image . open ( StringIO . StringIO ( image_data ) ) <EOL> pixel_data = list ( im . getdata ( ) ) <EOL> random . shuffle ( pixel_data ) <EOL> scrambled_image = im . copy ( ) <EOL> scrambled_image . putdata ( pixel_data ) <EOL> output_image_io = StringIO . StringIO ( ) <EOL> scrambled_image . save ( output_image_io , im . format ) <EOL> output_image_data = output_image_io . getvalue ( ) <EOL> output_image_data . encode ( encoding = '<STR_LIT>' ) <EOL> response = copy . deepcopy ( response ) <EOL> response . set_data ( output_image_data ) <EOL> except Exception : <EOL> pass <EOL> return response <EOL> class DetailedHTTPResponse ( httplib . HTTPResponse ) : <EOL> """<STR_LIT>""" <EOL> def read_chunks ( self ) : <EOL> """<STR_LIT>""" <EOL> buf = [ ] <EOL> chunks = [ ] <EOL> delays = [ ] <EOL> if not self . chunked : <EOL> chunks . append ( self . read ( ) ) <EOL> delays . append ( <NUM_LIT:0> ) <EOL> else : <EOL> start = TIMER ( ) <EOL> try : <EOL> while True : <EOL> line = self . fp . readline ( ) <EOL> chunk_size = self . _read_chunk_size ( line ) <EOL> if chunk_size is None : <EOL> raise httplib . IncompleteRead ( '<STR_LIT>' . join ( chunks ) ) <EOL> if chunk_size == <NUM_LIT:0> : <EOL> break <EOL> delays . append ( TIMER ( ) - start ) <EOL> chunks . append ( self . _safe_read ( chunk_size ) ) <EOL> self . _safe_read ( <NUM_LIT:2> ) <EOL> start = TIMER ( ) <EOL> while True : <EOL> line = self . fp . readline ( ) <EOL> if not line or line == '<STR_LIT:\r\n>' : <EOL> break <EOL> finally : <EOL> self . close ( ) <EOL> return chunks , delays <EOL> @ classmethod <EOL> def _read_chunk_size ( cls , line ) : <EOL> chunk_extensions_pos = line . find ( '<STR_LIT:;>' ) <EOL> if chunk_extensions_pos != - <NUM_LIT:1> : <EOL> line = line [ : chunk_extensions_pos ] <EOL> try : <EOL> chunk_size = int ( line , <NUM_LIT:16> ) <EOL> except ValueError : <EOL> return None <EOL> return chunk_size <EOL> class DetailedHTTPConnection ( httplib . HTTPConnection ) : <EOL> """<STR_LIT>""" <EOL> response_class = DetailedHTTPResponse <EOL> class DetailedHTTPSResponse ( DetailedHTTPResponse ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class DetailedHTTPSConnection ( httplib . HTTPSConnection ) : <EOL> """<STR_LIT>""" <EOL> response_class = DetailedHTTPSResponse <EOL> def __init__ ( self , host , port ) : <EOL> if hasattr ( ssl , '<STR_LIT>' ) : <EOL> httplib . HTTPSConnection . __init__ ( <EOL> self , host = host , port = port , context = ssl . _create_unverified_context ( ) ) <EOL> else : <EOL> httplib . HTTPSConnection . __init__ ( self , host = host , port = port ) <EOL> class RealHttpFetch ( object ) : <EOL> def __init__ ( self , real_dns_lookup ) : <EOL> """<STR_LIT>""" <EOL> self . _real_dns_lookup = real_dns_lookup <EOL> @ staticmethod <EOL> def _GetHeaderNameValue ( header ) : <EOL> """<STR_LIT>""" <EOL> i = header . find ( '<STR_LIT::>' ) <EOL> if i > <NUM_LIT:0> : <EOL> return ( header [ : i ] . lower ( ) , header [ i + <NUM_LIT:1> : ] . strip ( ) ) <EOL> return None <EOL> @ staticmethod <EOL> def _ToTuples ( headers ) : <EOL> """<STR_LIT>""" <EOL> all_headers = [ ] <EOL> for line in headers : <EOL> if line [ <NUM_LIT:0> ] in '<STR_LIT>' : <EOL> if not all_headers : <EOL> logging . warning ( <EOL> '<STR_LIT>' , line ) <EOL> continue <EOL> name , value = all_headers . pop ( ) <EOL> value += '<STR_LIT>' + line . strip ( ) <EOL> else : <EOL> name_value = RealHttpFetch . _GetHeaderNameValue ( line ) <EOL> if not name_value : <EOL> logging . warning ( <EOL> '<STR_LIT>' , line ) <EOL> continue <EOL> name , value = name_value <EOL> all_headers . append ( ( name , value ) ) <EOL> return all_headers <EOL> @ staticmethod <EOL> def _get_request_host_port ( request ) : <EOL> host_parts = request . host . split ( '<STR_LIT::>' ) <EOL> host = host_parts [ <NUM_LIT:0> ] <EOL> port = int ( host_parts [ <NUM_LIT:1> ] ) if len ( host_parts ) == <NUM_LIT:2> else None <EOL> return host , port <EOL> @ staticmethod <EOL> def _get_system_proxy ( is_ssl ) : <EOL> return platformsettings . get_system_proxy ( is_ssl ) <EOL> def _get_connection ( self , request_host , request_port , is_ssl ) : <EOL> """<STR_LIT>""" <EOL> connection_host = request_host <EOL> connection_port = request_port <EOL> system_proxy = self . _get_system_proxy ( is_ssl ) <EOL> if system_proxy : <EOL> connection_host = system_proxy . host <EOL> connection_port = system_proxy . port <EOL> connection_ip = self . _real_dns_lookup ( connection_host ) <EOL> if not connection_ip : <EOL> logging . critical ( '<STR_LIT>' , connection_host ) <EOL> return None <EOL> if is_ssl : <EOL> connection = DetailedHTTPSConnection ( connection_ip , connection_port ) <EOL> if system_proxy : <EOL> connection . set_tunnel ( request_host , request_port ) <EOL> else : <EOL> connection = DetailedHTTPConnection ( connection_ip , connection_port ) <EOL> return connection <EOL> def __call__ ( self , request ) : <EOL> """<STR_LIT>""" <EOL> logging . debug ( '<STR_LIT>' , request . host , request . full_path ) <EOL> request_host , request_port = self . _get_request_host_port ( request ) <EOL> retries = <NUM_LIT:3> <EOL> while True : <EOL> try : <EOL> connection = self . _get_connection ( <EOL> request_host , request_port , request . is_ssl ) <EOL> connect_start = TIMER ( ) <EOL> connection . connect ( ) <EOL> connect_delay = int ( ( TIMER ( ) - connect_start ) * <NUM_LIT:1000> ) <EOL> start = TIMER ( ) <EOL> connection . request ( <EOL> request . command , <EOL> request . full_path , <EOL> request . request_body , <EOL> request . headers ) <EOL> response = connection . getresponse ( ) <EOL> headers_delay = int ( ( TIMER ( ) - start ) * <NUM_LIT:1000> ) <EOL> chunks , chunk_delays = response . read_chunks ( ) <EOL> delays = { <EOL> '<STR_LIT>' : connect_delay , <EOL> '<STR_LIT>' : headers_delay , <EOL> '<STR_LIT:data>' : chunk_delays <EOL> } <EOL> archived_http_response = httparchive . ArchivedHttpResponse ( <EOL> response . version , <EOL> response . status , <EOL> response . reason , <EOL> RealHttpFetch . _ToTuples ( response . msg . headers ) , <EOL> chunks , <EOL> delays ) <EOL> return archived_http_response <EOL> except Exception , e : <EOL> if retries : <EOL> retries -= <NUM_LIT:1> <EOL> logging . warning ( '<STR_LIT>' , request , repr ( e ) ) <EOL> continue <EOL> logging . critical ( '<STR_LIT>' , request , repr ( e ) ) <EOL> return None <EOL> class RecordHttpArchiveFetch ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , http_archive , real_dns_lookup , inject_script ) : <EOL> """<STR_LIT>""" <EOL> self . http_archive = http_archive <EOL> self . real_http_fetch = RealHttpFetch ( real_dns_lookup ) <EOL> self . inject_script = inject_script <EOL> def __call__ ( self , request ) : <EOL> """<STR_LIT>""" <EOL> if request in self . http_archive : <EOL> logging . debug ( '<STR_LIT>' , request ) <EOL> response = self . http_archive [ request ] <EOL> else : <EOL> response = self . real_http_fetch ( request ) <EOL> if response is None : <EOL> return None <EOL> self . http_archive [ request ] = response <EOL> if self . inject_script : <EOL> response = _InjectScripts ( response , self . inject_script ) <EOL> logging . debug ( '<STR_LIT>' , request ) <EOL> return response <EOL> class ReplayHttpArchiveFetch ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , http_archive , real_dns_lookup , inject_script , <EOL> use_diff_on_unknown_requests = False , <EOL> use_closest_match = False , scramble_images = False ) : <EOL> """<STR_LIT>""" <EOL> self . http_archive = http_archive <EOL> self . inject_script = inject_script <EOL> self . use_diff_on_unknown_requests = use_diff_on_unknown_requests <EOL> self . use_closest_match = use_closest_match <EOL> self . scramble_images = scramble_images <EOL> self . real_http_fetch = RealHttpFetch ( real_dns_lookup ) <EOL> def __call__ ( self , request ) : <EOL> """<STR_LIT>""" <EOL> if request . host . startswith ( '<STR_LIT>' ) : <EOL> return self . real_http_fetch ( request ) <EOL> response = self . http_archive . get ( request ) <EOL> if self . use_closest_match and not response : <EOL> closest_request = self . http_archive . find_closest_request ( <EOL> request , use_path = True ) <EOL> if closest_request : <EOL> response = self . http_archive . get ( closest_request ) <EOL> if response : <EOL> logging . info ( '<STR_LIT>' , <EOL> request , closest_request ) <EOL> if not response : <EOL> reason = str ( request ) <EOL> if self . use_diff_on_unknown_requests : <EOL> diff = self . http_archive . diff ( request ) <EOL> if diff : <EOL> reason += ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % diff ) <EOL> logging . warning ( '<STR_LIT>' , reason ) <EOL> else : <EOL> if self . inject_script : <EOL> response = _InjectScripts ( response , self . inject_script ) <EOL> if self . scramble_images : <EOL> response = _ScrambleImages ( response ) <EOL> return response <EOL> class ControllableHttpArchiveFetch ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , http_archive , real_dns_lookup , <EOL> inject_script , use_diff_on_unknown_requests , <EOL> use_record_mode , use_closest_match , scramble_images ) : <EOL> """<STR_LIT>""" <EOL> self . http_archive = http_archive <EOL> self . record_fetch = RecordHttpArchiveFetch ( <EOL> http_archive , real_dns_lookup , inject_script ) <EOL> self . replay_fetch = ReplayHttpArchiveFetch ( <EOL> http_archive , real_dns_lookup , inject_script , <EOL> use_diff_on_unknown_requests , use_closest_match , scramble_images ) <EOL> if use_record_mode : <EOL> self . SetRecordMode ( ) <EOL> else : <EOL> self . SetReplayMode ( ) <EOL> def SetRecordMode ( self ) : <EOL> self . fetch = self . record_fetch <EOL> self . is_record_mode = True <EOL> def SetReplayMode ( self ) : <EOL> self . fetch = self . replay_fetch <EOL> self . is_record_mode = False <EOL> def __call__ ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> return self . fetch ( * args , ** kwargs ) </s>
