<s> """<STR_LIT>""" <EOL> _successfully_imported_fancy_urllib = False <EOL> _fancy_urllib_InvalidCertException = None <EOL> _fancy_urllib_SSLError = None <EOL> try : <EOL> import fancy_urllib <EOL> _successfully_imported_fancy_urllib = True <EOL> _fancy_urllib_InvalidCertException = fancy_urllib . InvalidCertificateException <EOL> _fancy_urllib_SSLError = fancy_urllib . SSLError <EOL> except ImportError : <EOL> pass <EOL> import gzip <EOL> import httplib <EOL> import logging <EOL> import os <EOL> import socket <EOL> import StringIO <EOL> import sys <EOL> import urllib <EOL> import urlparse <EOL> from google . appengine . api import apiproxy_stub <EOL> from google . appengine . api import urlfetch <EOL> from google . appengine . api import urlfetch_errors <EOL> from google . appengine . api import urlfetch_service_pb <EOL> from google . appengine . runtime import apiproxy_errors <EOL> MAX_REQUEST_SIZE = <NUM_LIT:10> << <NUM_LIT:20> <EOL> MAX_RESPONSE_SIZE = <NUM_LIT:2> ** <NUM_LIT> <EOL> MAX_REDIRECTS = urlfetch . MAX_REDIRECTS <EOL> REDIRECT_STATUSES = frozenset ( [ <EOL> httplib . MOVED_PERMANENTLY , <EOL> httplib . FOUND , <EOL> httplib . SEE_OTHER , <EOL> httplib . TEMPORARY_REDIRECT , <EOL> ] ) <EOL> PRESERVE_ON_REDIRECT = frozenset ( [ '<STR_LIT:GET>' , '<STR_LIT>' ] ) <EOL> _API_CALL_DEADLINE = <NUM_LIT> <EOL> _API_CALL_VALIDATE_CERTIFICATE_DEFAULT = False <EOL> _CONNECTION_SUPPORTS_TIMEOUT = sys . version_info >= ( <NUM_LIT:2> , <NUM_LIT:6> ) <EOL> _CONNECTION_SUPPORTS_SSL_TUNNEL = sys . version_info >= ( <NUM_LIT:2> , <NUM_LIT:6> ) <EOL> _MAX_REQUEST_SIZE = <NUM_LIT> <EOL> _UNTRUSTED_REQUEST_HEADERS = frozenset ( [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:host>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] ) <EOL> _MAX_URL_LENGTH = <NUM_LIT> <EOL> def _CanValidateCerts ( ) : <EOL> return ( _successfully_imported_fancy_urllib and <EOL> fancy_urllib . can_validate_certs ( ) ) <EOL> def _SetupSSL ( path ) : <EOL> global CERT_PATH <EOL> if os . path . exists ( path ) : <EOL> CERT_PATH = path <EOL> else : <EOL> CERT_PATH = None <EOL> logging . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' , path ) <EOL> if not _CanValidateCerts ( ) : <EOL> logging . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> _SetupSSL ( os . path . normpath ( os . path . join ( os . path . dirname ( __file__ ) , '<STR_LIT:..>' , '<STR_LIT:..>' , <EOL> '<STR_LIT:..>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) ) ) <EOL> def _IsAllowedPort ( port ) : <EOL> if port is None : <EOL> return True <EOL> try : <EOL> port = int ( port ) <EOL> except ValueError , e : <EOL> return False <EOL> if ( ( port >= <NUM_LIT> and port <= <NUM_LIT> ) or <EOL> ( port >= <NUM_LIT> and port <= <NUM_LIT> ) or <EOL> port >= <NUM_LIT> ) : <EOL> return True <EOL> return False <EOL> def _IsLocalhost ( host ) : <EOL> """<STR_LIT>""" <EOL> return host . startswith ( '<STR_LIT:localhost>' ) or host . startswith ( '<STR_LIT:127.0.0.1>' ) <EOL> class URLFetchServiceStub ( apiproxy_stub . APIProxyStub ) : <EOL> """<STR_LIT>""" <EOL> THREADSAFE = True <EOL> def __init__ ( self , <EOL> service_name = '<STR_LIT>' , <EOL> urlmatchers_to_fetch_functions = None ) : <EOL> """<STR_LIT>""" <EOL> super ( URLFetchServiceStub , self ) . __init__ ( service_name , <EOL> max_request_size = MAX_REQUEST_SIZE ) <EOL> self . _urlmatchers_to_fetch_functions = urlmatchers_to_fetch_functions or [ ] <EOL> def _Dynamic_Fetch ( self , request , response ) : <EOL> """<STR_LIT>""" <EOL> if len ( request . url ( ) ) >= _MAX_URL_LENGTH : <EOL> logging . error ( '<STR_LIT>' % request . url ( ) [ : <NUM_LIT:50> ] ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL ) <EOL> ( protocol , host , path , query , fragment ) = urlparse . urlsplit ( request . url ( ) ) <EOL> payload = None <EOL> if request . method ( ) == urlfetch_service_pb . URLFetchRequest . GET : <EOL> method = '<STR_LIT:GET>' <EOL> elif request . method ( ) == urlfetch_service_pb . URLFetchRequest . POST : <EOL> method = '<STR_LIT:POST>' <EOL> payload = request . payload ( ) <EOL> elif request . method ( ) == urlfetch_service_pb . URLFetchRequest . HEAD : <EOL> method = '<STR_LIT>' <EOL> elif request . method ( ) == urlfetch_service_pb . URLFetchRequest . PUT : <EOL> method = '<STR_LIT>' <EOL> payload = request . payload ( ) <EOL> elif request . method ( ) == urlfetch_service_pb . URLFetchRequest . DELETE : <EOL> method = '<STR_LIT>' <EOL> elif request . method ( ) == urlfetch_service_pb . URLFetchRequest . PATCH : <EOL> method = '<STR_LIT>' <EOL> payload = request . payload ( ) <EOL> else : <EOL> logging . error ( '<STR_LIT>' , request . method ( ) ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL ) <EOL> if payload is not None and len ( payload ) > _MAX_REQUEST_SIZE : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . PAYLOAD_TOO_LARGE ) <EOL> if not ( protocol == '<STR_LIT:http>' or protocol == '<STR_LIT>' ) : <EOL> logging . error ( '<STR_LIT>' , protocol ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL ) <EOL> if not host : <EOL> logging . error ( '<STR_LIT>' ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL ) <EOL> self . _SanitizeHttpHeaders ( _UNTRUSTED_REQUEST_HEADERS , <EOL> request . header_list ( ) ) <EOL> deadline = _API_CALL_DEADLINE <EOL> if request . has_deadline ( ) : <EOL> deadline = request . deadline ( ) <EOL> validate_certificate = _API_CALL_VALIDATE_CERTIFICATE_DEFAULT <EOL> if request . has_mustvalidateservercertificate ( ) : <EOL> validate_certificate = request . mustvalidateservercertificate ( ) <EOL> fetch_function = self . _GetFetchFunction ( request . url ( ) ) <EOL> fetch_function ( request . url ( ) , payload , method , <EOL> request . header_list ( ) , request , response , <EOL> follow_redirects = request . followredirects ( ) , <EOL> deadline = deadline , <EOL> validate_certificate = validate_certificate ) <EOL> def _GetFetchFunction ( self , url ) : <EOL> """<STR_LIT>""" <EOL> for urlmatcher , fetch_function in self . _urlmatchers_to_fetch_functions : <EOL> if urlmatcher ( url ) : <EOL> return fetch_function <EOL> return self . _RetrieveURL <EOL> @ staticmethod <EOL> def _RetrieveURL ( url , payload , method , headers , request , response , <EOL> follow_redirects = True , deadline = _API_CALL_DEADLINE , <EOL> validate_certificate = _API_CALL_VALIDATE_CERTIFICATE_DEFAULT ) : <EOL> """<STR_LIT>""" <EOL> last_protocol = '<STR_LIT>' <EOL> last_host = '<STR_LIT>' <EOL> if isinstance ( payload , unicode ) : <EOL> payload = payload . encode ( '<STR_LIT:utf-8>' ) <EOL> for redirect_number in xrange ( MAX_REDIRECTS + <NUM_LIT:1> ) : <EOL> parsed = urlparse . urlsplit ( url ) <EOL> protocol , host , path , query , fragment = parsed <EOL> port = urllib . splitport ( urllib . splituser ( host ) [ <NUM_LIT:1> ] ) [ <NUM_LIT:1> ] <EOL> if not _IsAllowedPort ( port ) : <EOL> logging . error ( <EOL> '<STR_LIT>' % <EOL> ( url , port ) ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL ) <EOL> if protocol and not host : <EOL> logging . error ( '<STR_LIT>' % url ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL ) <EOL> if not host and not protocol : <EOL> host = last_host <EOL> protocol = last_protocol <EOL> adjusted_headers = { <EOL> '<STR_LIT>' : <EOL> ( '<STR_LIT>' <EOL> % os . getenv ( '<STR_LIT>' ) ) , <EOL> '<STR_LIT>' : host , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> if payload is not None : <EOL> adjusted_headers [ '<STR_LIT>' ] = str ( len ( payload ) ) <EOL> if method == '<STR_LIT:POST>' and payload : <EOL> adjusted_headers [ '<STR_LIT:Content-Type>' ] = '<STR_LIT>' <EOL> passthrough_content_encoding = False <EOL> for header in headers : <EOL> if header . key ( ) . title ( ) . lower ( ) == '<STR_LIT>' : <EOL> adjusted_headers [ '<STR_LIT>' ] = ( <EOL> '<STR_LIT>' % <EOL> ( header . value ( ) , adjusted_headers [ '<STR_LIT>' ] ) ) <EOL> else : <EOL> if header . key ( ) . lower ( ) == '<STR_LIT>' : <EOL> passthrough_content_encoding = True <EOL> adjusted_headers [ header . key ( ) . title ( ) ] = header . value ( ) <EOL> if payload is not None : <EOL> escaped_payload = payload . encode ( '<STR_LIT>' ) <EOL> else : <EOL> escaped_payload = '<STR_LIT>' <EOL> logging . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> host , url , escaped_payload , adjusted_headers ) <EOL> try : <EOL> proxy_host = None <EOL> if protocol == '<STR_LIT:http>' : <EOL> connection_class = httplib . HTTPConnection <EOL> default_port = <NUM_LIT> <EOL> if os . environ . get ( '<STR_LIT>' ) and not _IsLocalhost ( host ) : <EOL> _ , proxy_host , _ , _ , _ = ( <EOL> urlparse . urlsplit ( os . environ . get ( '<STR_LIT>' ) ) ) <EOL> elif protocol == '<STR_LIT>' : <EOL> if ( validate_certificate and _CanValidateCerts ( ) and <EOL> CERT_PATH ) : <EOL> connection_class = fancy_urllib . create_fancy_connection ( <EOL> ca_certs = CERT_PATH ) <EOL> else : <EOL> connection_class = httplib . HTTPSConnection <EOL> default_port = <NUM_LIT> <EOL> if ( _CONNECTION_SUPPORTS_SSL_TUNNEL and <EOL> os . environ . get ( '<STR_LIT>' ) and not _IsLocalhost ( host ) ) : <EOL> _ , proxy_host , _ , _ , _ = ( <EOL> urlparse . urlsplit ( os . environ . get ( '<STR_LIT>' ) ) ) <EOL> else : <EOL> error_msg = '<STR_LIT>' % protocol <EOL> logging . error ( error_msg ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . INVALID_URL , error_msg ) <EOL> connection_kwargs = ( <EOL> { '<STR_LIT>' : deadline } if _CONNECTION_SUPPORTS_TIMEOUT else { } ) <EOL> if proxy_host : <EOL> proxy_address , _ , proxy_port = proxy_host . partition ( '<STR_LIT::>' ) <EOL> connection = connection_class ( <EOL> proxy_address , proxy_port if proxy_port else default_port , <EOL> ** connection_kwargs ) <EOL> full_path = urlparse . urlunsplit ( ( protocol , host , path , query , '<STR_LIT>' ) ) <EOL> if protocol == '<STR_LIT>' : <EOL> connection . set_tunnel ( host ) <EOL> else : <EOL> connection = connection_class ( host , ** connection_kwargs ) <EOL> full_path = urlparse . urlunsplit ( ( '<STR_LIT>' , '<STR_LIT>' , path , query , '<STR_LIT>' ) ) <EOL> last_protocol = protocol <EOL> last_host = host <EOL> if not _CONNECTION_SUPPORTS_TIMEOUT : <EOL> orig_timeout = socket . getdefaulttimeout ( ) <EOL> try : <EOL> if not _CONNECTION_SUPPORTS_TIMEOUT : <EOL> socket . setdefaulttimeout ( deadline ) <EOL> connection . request ( method , full_path , payload , adjusted_headers ) <EOL> http_response = connection . getresponse ( ) <EOL> if method == '<STR_LIT>' : <EOL> http_response_data = '<STR_LIT>' <EOL> else : <EOL> http_response_data = http_response . read ( ) <EOL> finally : <EOL> if not _CONNECTION_SUPPORTS_TIMEOUT : <EOL> socket . setdefaulttimeout ( orig_timeout ) <EOL> connection . close ( ) <EOL> except _fancy_urllib_InvalidCertException , e : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . SSL_CERTIFICATE_ERROR , <EOL> str ( e ) ) <EOL> except _fancy_urllib_SSLError , e : <EOL> app_error = ( <EOL> urlfetch_service_pb . URLFetchServiceError . DEADLINE_EXCEEDED <EOL> if '<STR_LIT>' in e . message else <EOL> urlfetch_service_pb . URLFetchServiceError . SSL_CERTIFICATE_ERROR ) <EOL> raise apiproxy_errors . ApplicationError ( app_error , str ( e ) ) <EOL> except socket . timeout , e : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . DEADLINE_EXCEEDED , str ( e ) ) <EOL> except ( httplib . error , socket . error , IOError ) , e : <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . FETCH_ERROR , str ( e ) ) <EOL> if http_response . status in REDIRECT_STATUSES and follow_redirects : <EOL> url = http_response . getheader ( '<STR_LIT>' , None ) <EOL> if url is None : <EOL> error_msg = '<STR_LIT>' <EOL> logging . error ( error_msg ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . MALFORMED_REPLY , <EOL> error_msg ) <EOL> if ( http_response . status != httplib . TEMPORARY_REDIRECT and <EOL> method not in PRESERVE_ON_REDIRECT ) : <EOL> logging . warn ( '<STR_LIT>' , <EOL> http_response . status , method ) <EOL> method = '<STR_LIT:GET>' <EOL> payload = None <EOL> else : <EOL> response . set_statuscode ( http_response . status ) <EOL> if ( http_response . getheader ( '<STR_LIT>' ) == '<STR_LIT>' and <EOL> not passthrough_content_encoding ) : <EOL> gzip_stream = StringIO . StringIO ( http_response_data ) <EOL> gzip_file = gzip . GzipFile ( fileobj = gzip_stream ) <EOL> http_response_data = gzip_file . read ( ) <EOL> response . set_content ( http_response_data [ : MAX_RESPONSE_SIZE ] ) <EOL> for header_key in http_response . msg . keys ( ) : <EOL> for header_value in http_response . msg . getheaders ( header_key ) : <EOL> if ( header_key . lower ( ) == '<STR_LIT>' and <EOL> header_value == '<STR_LIT>' and <EOL> not passthrough_content_encoding ) : <EOL> continue <EOL> if header_key . lower ( ) == '<STR_LIT>' and method != '<STR_LIT>' : <EOL> header_value = str ( len ( response . content ( ) ) ) <EOL> header_proto = response . add_header ( ) <EOL> header_proto . set_key ( header_key ) <EOL> header_proto . set_value ( header_value ) <EOL> if len ( http_response_data ) > MAX_RESPONSE_SIZE : <EOL> response . set_contentwastruncated ( True ) <EOL> if request . url ( ) != url : <EOL> response . set_finalurl ( url ) <EOL> break <EOL> else : <EOL> error_msg = '<STR_LIT>' <EOL> logging . error ( error_msg ) <EOL> raise apiproxy_errors . ApplicationError ( <EOL> urlfetch_service_pb . URLFetchServiceError . TOO_MANY_REDIRECTS , <EOL> error_msg ) <EOL> def _SanitizeHttpHeaders ( self , untrusted_headers , headers ) : <EOL> """<STR_LIT>""" <EOL> prohibited_headers = [ h . key ( ) for h in headers <EOL> if h . key ( ) . lower ( ) in untrusted_headers ] <EOL> if prohibited_headers : <EOL> logging . warn ( '<STR_LIT>' , <EOL> prohibited_headers ) <EOL> for index in reversed ( xrange ( len ( headers ) ) ) : <EOL> if headers [ index ] . key ( ) . lower ( ) in untrusted_headers : <EOL> del headers [ index ] </s>
