<s> """<STR_LIT>""" <EOL> import sys <EOL> import cgi <EOL> import httplib <EOL> import logging <EOL> import urllib <EOL> import urllib2 <EOL> from framework . dependency_management . dependency_resolver import BaseComponent <EOL> from framework . dependency_management . interfaces import RequesterInterface <EOL> from framework . http import transaction <EOL> from framework . lib . general import * <EOL> class MyHTTPConnection ( httplib . HTTPConnection ) : <EOL> def send ( self , s ) : <EOL> global raw_request <EOL> raw_request . append ( s ) <EOL> httplib . HTTPConnection . send ( self , s ) <EOL> class MyHTTPHandler ( urllib2 . HTTPHandler ) : <EOL> def http_open ( self , req ) : <EOL> try : <EOL> return self . do_open ( MyHTTPConnection , req ) <EOL> except KeyboardInterrupt : <EOL> raise KeyboardInterrupt <EOL> except Exception , e : <EOL> return '<STR_LIT>' <EOL> class MyHTTPSConnection ( httplib . HTTPSConnection ) : <EOL> def send ( self , s ) : <EOL> global raw_request <EOL> raw_request . append ( s ) <EOL> httplib . HTTPSConnection . send ( self , s ) <EOL> class MyHTTPSHandler ( urllib2 . HTTPSHandler ) : <EOL> def https_open ( self , req ) : <EOL> try : <EOL> return self . do_open ( MyHTTPSConnection , req ) <EOL> except KeyboardInterrupt : <EOL> raise KeyboardInterrupt <EOL> except Exception , e : <EOL> return '<STR_LIT>' <EOL> class SmartRedirectHandler ( urllib2 . HTTPRedirectHandler ) : <EOL> def http_error_301 ( self , req , fp , code , msg , headers ) : <EOL> result = urllib2 . HTTPRedirectHandler . http_error_301 ( <EOL> self , req , fp , code , msg , headers ) <EOL> result . status = code <EOL> return result <EOL> def http_error_302 ( self , req , fp , code , msg , headers ) : <EOL> result = urllib2 . HTTPRedirectHandler . http_error_302 ( <EOL> self , req , fp , code , msg , headers ) <EOL> result . status = code <EOL> return result <EOL> class Requester ( BaseComponent , RequesterInterface ) : <EOL> COMPONENT_NAME = "<STR_LIT>" <EOL> def __init__ ( self , proxy ) : <EOL> self . register_in_service_locator ( ) <EOL> self . db_config = self . get_component ( "<STR_LIT>" ) <EOL> self . target = self . get_component ( "<STR_LIT:target>" ) <EOL> self . transaction = self . get_component ( "<STR_LIT>" ) <EOL> self . url_manager = self . get_component ( "<STR_LIT>" ) <EOL> self . error_handler = self . get_component ( "<STR_LIT>" ) <EOL> self . plugin_handler = self . get_component ( "<STR_LIT>" ) <EOL> self . timer = self . get_component ( "<STR_LIT>" ) <EOL> self . http_transaction = None <EOL> self . Headers = { '<STR_LIT>' : self . db_config . Get ( '<STR_LIT>' ) } <EOL> self . RequestCountRefused = <NUM_LIT:0> <EOL> self . RequestCountTotal = <NUM_LIT:0> <EOL> self . LogTransactions = False <EOL> self . Proxy = proxy <EOL> if proxy is None : <EOL> logging . debug ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> self . Opener = urllib2 . build_opener ( <EOL> MyHTTPHandler , <EOL> MyHTTPSHandler , <EOL> SmartRedirectHandler ) <EOL> else : <EOL> logging . debug ( "<STR_LIT>" ) <EOL> ip , port = proxy <EOL> proxy_conf = { <EOL> '<STR_LIT:http>' : '<STR_LIT>' + ip + "<STR_LIT::>" + port , <EOL> '<STR_LIT>' : '<STR_LIT>' + ip + "<STR_LIT::>" + port } <EOL> proxy_handler = urllib2 . ProxyHandler ( proxy_conf ) <EOL> self . Opener = urllib2 . build_opener ( <EOL> proxy_handler , <EOL> MyHTTPHandler , <EOL> MyHTTPSHandler , <EOL> SmartRedirectHandler ) <EOL> urllib2 . install_opener ( self . Opener ) <EOL> def log_transactions ( self , log_transactions = True ) : <EOL> backup = self . LogTransactions <EOL> self . LogTransactions = log_transactions <EOL> return backup <EOL> def NeedToAskBeforeRequest ( self ) : <EOL> return not self . plugin_handler . NormalRequestsAllowed ( ) <EOL> def IsTransactionAlreadyAdded ( self , url ) : <EOL> return self . transaction . IsTransactionAlreadyAdded ( { '<STR_LIT:url>' : url . strip ( ) } ) <EOL> def is_request_possible ( self ) : <EOL> return self . plugin_handler . RequestsPossible ( ) <EOL> def ProxyCheck ( self ) : <EOL> if self . Proxy is not None and self . is_request_possible ( ) : <EOL> url = self . db_config . Get ( '<STR_LIT>' ) <EOL> refused_before = self . RequestCountRefused <EOL> cprint ( <EOL> "<STR_LIT>" ) <EOL> log_setting_backup = False <EOL> if self . IsTransactionAlreadyAdded ( url ) : <EOL> log_setting_backup = self . log_transactions ( False ) <EOL> if log_setting_backup : <EOL> self . log_transactions ( log_setting_backup ) <EOL> refused_after = self . RequestCountRefused <EOL> if refused_before < refused_after : <EOL> return [ <EOL> False , <EOL> "<STR_LIT>" "<STR_LIT>" ] <EOL> else : <EOL> return [ <EOL> True , <EOL> "<STR_LIT>" ] <EOL> return [ <EOL> True , <EOL> "<STR_LIT>" "<STR_LIT>" ] <EOL> def GetHeaders ( self ) : <EOL> return self . Headers <EOL> def SetHeaders ( self , headers ) : <EOL> self . Headers = headers <EOL> def SetHeader ( self , header , value ) : <EOL> self . Headers [ header ] = value <EOL> def StringToDict ( self , string ) : <EOL> dict = defaultdict ( list ) <EOL> count = <NUM_LIT:0> <EOL> prev_item = '<STR_LIT>' <EOL> for item in string . strip ( ) . split ( '<STR_LIT:=>' ) : <EOL> if count % <NUM_LIT:2> == <NUM_LIT:1> : <EOL> dict [ prev_item ] = item <EOL> else : <EOL> dict [ item ] = '<STR_LIT>' <EOL> prev_item = item <EOL> count += <NUM_LIT:1> <EOL> return dict <EOL> def DerivePOSTToStr ( self , post = None ) : <EOL> post = self . DerivePOST ( post ) <EOL> if post is None : <EOL> return '<STR_LIT>' <EOL> return post <EOL> def DerivePOST ( self , post = None ) : <EOL> if '<STR_LIT>' == post : <EOL> post = None <EOL> if post is not None : <EOL> if isinstance ( post , str ) or isinstance ( post , unicode ) : <EOL> post = self . StringToDict ( post ) <EOL> post = urllib . urlencode ( post ) <EOL> return post <EOL> def perform_request ( self , request ) : <EOL> return urllib2 . urlopen ( request ) <EOL> def set_succesful_transaction ( self , raw_request , response ) : <EOL> return self . http_transaction . SetTransaction ( True , raw_request [ <NUM_LIT:0> ] , response ) <EOL> def log_transaction ( self ) : <EOL> self . transaction . LogTransaction ( self . http_transaction ) <EOL> def Request ( self , url , method = None , post = None ) : <EOL> global raw_request <EOL> url = str ( url ) <EOL> raw_request = [ ] <EOL> post = self . DerivePOST ( post ) <EOL> method = DeriveHTTPMethod ( method , post ) <EOL> url = url . strip ( ) <EOL> request = urllib2 . Request ( url , post , self . Headers ) <EOL> if method is not None : <EOL> request . get_method = lambda : method <EOL> self . http_transaction = transaction . HTTP_Transaction ( self . timer ) <EOL> self . http_transaction . Start ( <EOL> url , <EOL> post , <EOL> method , <EOL> self . target . IsInScopeURL ( url ) ) <EOL> self . RequestCountTotal += <NUM_LIT:1> <EOL> try : <EOL> response = self . perform_request ( request ) <EOL> self . set_succesful_transaction ( raw_request , response ) <EOL> except urllib2 . HTTPError , Error : <EOL> self . http_transaction . SetTransaction ( False , raw_request [ <NUM_LIT:0> ] , Error ) <EOL> except urllib2 . URLError , Error : <EOL> err_message = self . ProcessHTTPErrorCode ( Error , url ) <EOL> self . http_transaction . SetError ( err_message ) <EOL> except IOError : <EOL> err_message = "<STR_LIT>" "<STR_LIT>" + url + "<STR_LIT:\n>" + str ( sys . exc_info ( ) ) <EOL> self . http_transaction . SetError ( err_message ) <EOL> if self . LogTransactions : <EOL> self . log_transaction ( ) <EOL> return self . http_transaction <EOL> def ProcessHTTPErrorCode ( self , error , url ) : <EOL> message = "<STR_LIT>" <EOL> if str ( error . reason ) . startswith ( "<STR_LIT>" ) : <EOL> message = "<STR_LIT>" + str ( error ) <EOL> self . RequestCountRefused += <NUM_LIT:1> <EOL> elif str ( error . reason ) . startswith ( "<STR_LIT>" ) : <EOL> self . error_handler . FrameworkAbort ( <EOL> "<STR_LIT>" + str ( error ) ) <EOL> else : <EOL> message = "<STR_LIT>" <EOL> log = logging . getLogger ( '<STR_LIT>' ) <EOL> log . info ( message ) <EOL> return message + "<STR_LIT>" + url + "<STR_LIT:\n>" + str ( sys . exc_info ( ) ) <EOL> def GET ( self , url ) : <EOL> return self . Request ( url ) <EOL> def POST ( self , url , data ) : <EOL> return self . Request ( url , '<STR_LIT:POST>' , data ) <EOL> def TRACE ( self , url ) : <EOL> return self . Request ( url , '<STR_LIT>' , None ) <EOL> def OPTIONS ( self , url ) : <EOL> return self . Request ( url , '<STR_LIT>' , None ) <EOL> def HEAD ( self , url ) : <EOL> return self . Request ( url , '<STR_LIT>' , None ) <EOL> def DEBUG ( self , url ) : <EOL> self . BackupHeaders ( ) <EOL> self . Headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> result = self . Request ( url , '<STR_LIT>' , None ) <EOL> self . RestoreHeaders ( ) <EOL> return result <EOL> def PUT ( self , url , content_type = '<STR_LIT>' ) : <EOL> self . BackupHeaders ( ) <EOL> self . Headers [ '<STR_LIT:Content-Type>' ] = content_type <EOL> self . Headers [ '<STR_LIT>' ] = "<STR_LIT:0>" <EOL> result = self . Request ( url , '<STR_LIT>' , None ) <EOL> self . RestoreHeaders ( ) <EOL> return result <EOL> def BackupHeaders ( self ) : <EOL> self . HeadersBackup = dict . copy ( self . Headers ) <EOL> def RestoreHeaders ( self ) : <EOL> self . Headers = dict . copy ( self . HeadersBackup ) <EOL> def GetTransaction ( self , use_cache , url , method = None , data = None ) : <EOL> criteria = { '<STR_LIT:url>' : url . strip ( ) } <EOL> if method is not None : <EOL> criteria [ '<STR_LIT>' ] = method <EOL> if data is not None : <EOL> criteria [ '<STR_LIT:data>' ] = self . DerivePOSTToStr ( data ) <EOL> if ( not use_cache or not self . transaction . IsTransactionAlreadyAdded ( criteria ) ) : <EOL> if method in [ '<STR_LIT>' , '<STR_LIT:GET>' , '<STR_LIT:POST>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> return self . Request ( url , method , data ) <EOL> elif method == '<STR_LIT>' : <EOL> return self . DEBUG ( url ) <EOL> elif method == '<STR_LIT>' : <EOL> return self . PUT ( url , data ) <EOL> else : <EOL> return self . transaction . GetFirst ( criteria ) <EOL> def GetTransactions ( self , <EOL> use_cache , <EOL> url_list , <EOL> method = None , <EOL> data = None , <EOL> unique = True ) : <EOL> transactions = [ ] <EOL> if unique : <EOL> url_list = set ( url_list ) <EOL> for url in url_list : <EOL> url = url . strip ( ) <EOL> if not url : <EOL> continue <EOL> if not self . url_manager . IsURL ( url ) : <EOL> self . error_handler . Add ( <EOL> "<STR_LIT>" + str ( url ) + "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> continue <EOL> transaction = self . GetTransaction ( use_cache , url , method = method , data = data ) <EOL> if transaction is not None : <EOL> transactions . append ( transaction ) <EOL> return transactions </s>
