<s> import re <EOL> import random <EOL> from twisted . internet import defer <EOL> from txtorcon . interface import StreamListenerMixin <EOL> from twisted . internet import reactor <EOL> from twisted . internet . endpoints import TCP4ClientEndpoint <EOL> from ooni . utils . trueheaders import TrueHeadersAgent , TrueHeadersSOCKS5Agent <EOL> from ooni . nettest import NetTestCase <EOL> from ooni . utils import log , base64Dict <EOL> from ooni . settings import config <EOL> from ooni . utils . net import BodyReceiver , StringProducer , userAgents <EOL> from ooni . utils . trueheaders import TrueHeaders <EOL> from ooni . errors import handleAllFailures <EOL> META_CHARSET_REGEXP = re . compile ( '<STR_LIT>' ) <EOL> class InvalidSocksProxyOption ( Exception ) : <EOL> pass <EOL> class StreamListener ( StreamListenerMixin ) : <EOL> def __init__ ( self , request ) : <EOL> self . request = request <EOL> def stream_succeeded ( self , stream ) : <EOL> host = self . request [ '<STR_LIT:url>' ] . split ( '<STR_LIT:/>' ) [ <NUM_LIT:2> ] <EOL> try : <EOL> if stream . target_host == host and len ( self . request [ '<STR_LIT>' ] ) == <NUM_LIT:1> : <EOL> self . request [ '<STR_LIT>' ] [ '<STR_LIT>' ] = stream . circuit . path [ - <NUM_LIT:1> ] . ip <EOL> self . request [ '<STR_LIT>' ] [ '<STR_LIT>' ] = stream . circuit . path [ - <NUM_LIT:1> ] . name <EOL> config . tor_state . stream_listeners . remove ( self ) <EOL> except : <EOL> log . err ( "<STR_LIT>" ) <EOL> def _representBody ( body ) : <EOL> body = body . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> decoded = False <EOL> charsets = [ '<STR_LIT:ascii>' , '<STR_LIT:utf-8>' ] <EOL> charset = META_CHARSET_REGEXP . search ( body , re . IGNORECASE ) <EOL> if charset : <EOL> charsets . insert ( <NUM_LIT:0> , charset . group ( <NUM_LIT:1> ) ) <EOL> for encoding in charsets : <EOL> try : <EOL> body = unicode ( body , encoding ) <EOL> decoded = True <EOL> break <EOL> except UnicodeDecodeError : <EOL> pass <EOL> if not decoded : <EOL> body = base64Dict ( body ) <EOL> return body <EOL> class HTTPTest ( NetTestCase ) : <EOL> """<STR_LIT>""" <EOL> name = "<STR_LIT>" <EOL> version = "<STR_LIT>" <EOL> randomizeUA = False <EOL> followRedirects = False <EOL> baseParameters = [ [ '<STR_LIT>' , '<STR_LIT:s>' , None , <EOL> '<STR_LIT>' ] ] <EOL> def _setUp ( self ) : <EOL> super ( HTTPTest , self ) . _setUp ( ) <EOL> try : <EOL> import OpenSSL <EOL> except : <EOL> log . err ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> self . control_agent = TrueHeadersSOCKS5Agent ( reactor , <EOL> proxyEndpoint = TCP4ClientEndpoint ( reactor , '<STR_LIT:127.0.0.1>' , <EOL> config . tor . socks_port ) ) <EOL> self . report [ '<STR_LIT>' ] = None <EOL> sockshost , socksport = ( None , None ) <EOL> if self . localOptions [ '<STR_LIT>' ] : <EOL> try : <EOL> sockshost , socksport = self . localOptions [ '<STR_LIT>' ] . split ( '<STR_LIT::>' ) <EOL> self . report [ '<STR_LIT>' ] = self . localOptions [ '<STR_LIT>' ] <EOL> except ValueError : <EOL> raise InvalidSocksProxyOption <EOL> socksport = int ( socksport ) <EOL> self . agent = TrueHeadersSOCKS5Agent ( reactor , <EOL> proxyEndpoint = TCP4ClientEndpoint ( reactor , sockshost , <EOL> socksport ) ) <EOL> else : <EOL> self . agent = TrueHeadersAgent ( reactor ) <EOL> self . report [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if self . followRedirects : <EOL> try : <EOL> from twisted . web . client import RedirectAgent <EOL> self . control_agent = RedirectAgent ( self . control_agent ) <EOL> self . agent = RedirectAgent ( self . agent ) <EOL> self . report [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> except : <EOL> log . err ( "<STR_LIT>" "<STR_LIT>" "<STR_LIT>" ) <EOL> self . processInputs ( ) <EOL> log . debug ( "<STR_LIT>" ) <EOL> def randomize_useragent ( self , request ) : <EOL> user_agent = random . choice ( userAgents ) <EOL> request [ '<STR_LIT>' ] [ '<STR_LIT>' ] = [ user_agent ] <EOL> def processInputs ( self ) : <EOL> pass <EOL> def addToReport ( self , request , response = None , response_body = None , failure_string = None ) : <EOL> """<STR_LIT>""" <EOL> def _representHeaders ( headers ) : <EOL> represented_headers = { } <EOL> for name , value in headers . getAllRawHeaders ( ) : <EOL> represented_headers [ name ] = value [ <NUM_LIT:0> ] <EOL> return represented_headers <EOL> log . debug ( "<STR_LIT>" % request ) <EOL> request_headers = TrueHeaders ( request [ '<STR_LIT>' ] ) <EOL> session = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : _representHeaders ( request_headers ) , <EOL> '<STR_LIT:body>' : request [ '<STR_LIT:body>' ] , <EOL> '<STR_LIT:url>' : request [ '<STR_LIT:url>' ] , <EOL> '<STR_LIT>' : request [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : request [ '<STR_LIT>' ] <EOL> } , <EOL> '<STR_LIT>' : None <EOL> } <EOL> if response : <EOL> if self . localOptions . get ( '<STR_LIT>' , <NUM_LIT:0> ) is <NUM_LIT:0> : <EOL> response_body = _representBody ( response_body ) <EOL> else : <EOL> response_body = '<STR_LIT>' <EOL> if ( config . privacy . includeip is False and config . probe_ip . address is not None and <EOL> ( isinstance ( response_body , str ) or isinstance ( response_body , unicode ) ) ) : <EOL> response_body = response_body . replace ( config . probe_ip . address , "<STR_LIT>" ) <EOL> session [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : _representHeaders ( response . headers ) , <EOL> '<STR_LIT:body>' : response_body , <EOL> '<STR_LIT:code>' : response . code <EOL> } <EOL> session [ '<STR_LIT>' ] = None <EOL> if failure_string : <EOL> session [ '<STR_LIT>' ] = failure_string <EOL> self . report [ '<STR_LIT>' ] . append ( session ) <EOL> def _processResponseBody ( self , response_body , request , response , body_processor ) : <EOL> log . debug ( "<STR_LIT>" ) <EOL> HTTPTest . addToReport ( self , request , response , response_body ) <EOL> if body_processor : <EOL> body_processor ( response_body ) <EOL> else : <EOL> self . processResponseBody ( response_body ) <EOL> response . body = response_body <EOL> return response <EOL> def _processResponseBodyFail ( self , failure , request , response ) : <EOL> failure_string = handleAllFailures ( failure ) <EOL> HTTPTest . addToReport ( self , request , response , <EOL> failure_string = failure_string ) <EOL> return response <EOL> def processResponseBody ( self , body ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def processResponseHeaders ( self , headers ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def processRedirect ( self , location ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def _cbResponse ( self , response , request , <EOL> headers_processor , body_processor ) : <EOL> """<STR_LIT>""" <EOL> if not response : <EOL> log . err ( "<STR_LIT>" % request ) <EOL> HTTPTest . addToReport ( self , request , response ) <EOL> return <EOL> else : <EOL> log . debug ( "<STR_LIT>" % response ) <EOL> if str ( response . code ) . startswith ( '<STR_LIT:3>' ) : <EOL> self . processRedirect ( response . headers . getRawHeaders ( '<STR_LIT>' ) [ <NUM_LIT:0> ] ) <EOL> response_headers_dict = list ( response . headers . getAllRawHeaders ( ) ) <EOL> if headers_processor : <EOL> headers_processor ( response_headers_dict ) <EOL> else : <EOL> self . processResponseHeaders ( response_headers_dict ) <EOL> try : <EOL> content_length = int ( response . headers . getRawHeaders ( '<STR_LIT>' ) [ <NUM_LIT:0> ] ) <EOL> except Exception : <EOL> content_length = None <EOL> finished = defer . Deferred ( ) <EOL> response . deliverBody ( BodyReceiver ( finished , content_length ) ) <EOL> finished . addCallback ( self . _processResponseBody , request , <EOL> response , body_processor ) <EOL> finished . addErrback ( self . _processResponseBodyFail , request , <EOL> response ) <EOL> return finished <EOL> def doRequest ( self , url , method = "<STR_LIT:GET>" , <EOL> headers = { } , body = None , headers_processor = None , <EOL> body_processor = None , use_tor = False ) : <EOL> """<STR_LIT>""" <EOL> if use_tor : <EOL> log . debug ( "<STR_LIT>" % url ) <EOL> agent = self . control_agent <EOL> else : <EOL> agent = self . agent <EOL> if self . localOptions [ '<STR_LIT>' ] : <EOL> log . debug ( "<STR_LIT>" % ( self . localOptions [ '<STR_LIT>' ] ) ) <EOL> log . debug ( "<STR_LIT>" % ( url , method , headers ) ) <EOL> request = { } <EOL> request [ '<STR_LIT>' ] = method <EOL> request [ '<STR_LIT:url>' ] = url <EOL> request [ '<STR_LIT>' ] = headers <EOL> request [ '<STR_LIT:body>' ] = body <EOL> request [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : None <EOL> } <EOL> if use_tor : <EOL> request [ '<STR_LIT>' ] [ '<STR_LIT>' ] = True <EOL> else : <EOL> request [ '<STR_LIT>' ] [ '<STR_LIT>' ] = False <EOL> if self . randomizeUA : <EOL> log . debug ( "<STR_LIT>" ) <EOL> self . randomize_useragent ( request ) <EOL> self . report [ '<STR_LIT>' ] = self . report . get ( '<STR_LIT>' , [ ] ) <EOL> if body : <EOL> body_producer = StringProducer ( request [ '<STR_LIT:body>' ] ) <EOL> else : <EOL> body_producer = None <EOL> headers = TrueHeaders ( request [ '<STR_LIT>' ] ) <EOL> def errback ( failure , request ) : <EOL> if request [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> log . err ( "<STR_LIT>" % request [ '<STR_LIT:url>' ] ) <EOL> else : <EOL> log . err ( "<STR_LIT>" % request [ '<STR_LIT:url>' ] ) <EOL> failure_string = handleAllFailures ( failure ) <EOL> self . addToReport ( request , failure_string = failure_string ) <EOL> return failure <EOL> if use_tor : <EOL> state = config . tor_state <EOL> if state : <EOL> state . add_stream_listener ( StreamListener ( request ) ) <EOL> d = agent . request ( request [ '<STR_LIT>' ] , request [ '<STR_LIT:url>' ] , headers , <EOL> body_producer ) <EOL> d . addErrback ( errback , request ) <EOL> d . addCallback ( self . _cbResponse , request , headers_processor , <EOL> body_processor ) <EOL> return d </s>
