<s> """<STR_LIT>""" <EOL> __author__ = "<STR_LIT>" <EOL> __date__ = "<STR_LIT>" <EOL> __copyright__ = "<STR_LIT>" <EOL> __license__ = "<STR_LIT>" <EOL> __contact__ = "<STR_LIT>" <EOL> __revision__ = '<STR_LIT>' <EOL> import logging <EOL> from optparse import OptionParser <EOL> import os <EOL> import sys <EOL> if sys . version_info [ <NUM_LIT:0> ] > <NUM_LIT:2> : <EOL> import http . cookiejar as cookiejar_ <EOL> import http . client as http_client_ <EOL> from urllib . request import Request as Request_ <EOL> from urllib . request import HTTPHandler as HTTPHandler_ <EOL> from urllib . request import HTTPCookieProcessor as HTTPCookieProcessor_ <EOL> from urllib . request import HTTPBasicAuthHandler as HTTPBasicAuthHandler_ <EOL> from urllib . request import HTTPPasswordMgrWithDefaultRealm as HTTPPasswordMgrWithDefaultRealm_ <EOL> from urllib . request import ProxyHandler as ProxyHandler_ <EOL> from urllib . error import HTTPError as HTTPError_ <EOL> import urllib . parse as urlparse_ <EOL> else : <EOL> import cookielib as cookiejar_ <EOL> import httplib as http_client_ <EOL> from urllib2 import Request as Request_ <EOL> from urllib2 import HTTPHandler as HTTPHandler_ <EOL> from urllib2 import HTTPCookieProcessor as HTTPCookieProcessor_ <EOL> from urllib2 import HTTPBasicAuthHandler as HTTPBasicAuthHandler_ <EOL> from urllib2 import HTTPPasswordMgrWithDefaultRealm as HTTPPasswordMgrWithDefaultRealm_ <EOL> from urllib2 import ProxyHandler as ProxyHandler_ <EOL> from urllib2 import HTTPError as HTTPError_ <EOL> import urlparse as urlparse_ <EOL> from ndg . httpsclient . urllib2_build_opener import build_opener <EOL> from ndg . httpsclient . https import HTTPSContextHandler <EOL> from ndg . httpsclient import ssl_context_util <EOL> log = logging . getLogger ( __name__ ) <EOL> class AccumulatingHTTPCookieProcessor ( HTTPCookieProcessor_ ) : <EOL> """<STR_LIT>""" <EOL> def http_request ( self , request ) : <EOL> """<STR_LIT>""" <EOL> COOKIE_HEADER_NAME = "<STR_LIT>" <EOL> tmp_request = Request_ ( request . get_full_url ( ) , request . data , { } , <EOL> request . origin_req_host , <EOL> request . unverifiable ) <EOL> self . cookiejar . add_cookie_header ( tmp_request ) <EOL> new_cookies = tmp_request . get_header ( COOKIE_HEADER_NAME ) <EOL> if new_cookies : <EOL> if request . has_header ( COOKIE_HEADER_NAME ) : <EOL> old_cookies = request . get_header ( COOKIE_HEADER_NAME ) <EOL> merged_cookies = '<STR_LIT>' . join ( [ old_cookies , new_cookies ] ) <EOL> request . add_unredirected_header ( COOKIE_HEADER_NAME , <EOL> merged_cookies ) <EOL> else : <EOL> request . add_unredirected_header ( COOKIE_HEADER_NAME , new_cookies ) <EOL> return request <EOL> https_request = http_request <EOL> class URLFetchError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> def fetch_from_url ( url , config , data = None , handlers = None ) : <EOL> """<STR_LIT>""" <EOL> return_code , return_message , response = open_url ( url , config , data = data , <EOL> handlers = handlers ) <EOL> if return_code and return_code == http_client_ . OK : <EOL> return_data = response . read ( ) <EOL> response . close ( ) <EOL> return return_data <EOL> else : <EOL> raise URLFetchError ( return_message ) <EOL> def fetch_from_url_to_file ( url , config , output_file , data = None , handlers = None ) : <EOL> """<STR_LIT>""" <EOL> return_code , return_message , response = open_url ( url , config , data = data , <EOL> handlers = handlers ) <EOL> if return_code == http_client_ . OK : <EOL> return_data = response . read ( ) <EOL> response . close ( ) <EOL> outfile = open ( output_file , "<STR_LIT:w>" ) <EOL> outfile . write ( return_data ) <EOL> outfile . close ( ) <EOL> return return_code , return_message , return_code == http_client_ . OK <EOL> def fetch_stream_from_url ( url , config , data = None , handlers = None ) : <EOL> """<STR_LIT>""" <EOL> return_code , return_message , response = open_url ( url , config , data = data , <EOL> handlers = handlers ) <EOL> if return_code and return_code == http_client_ . OK : <EOL> return response <EOL> else : <EOL> raise URLFetchError ( return_message ) <EOL> def open_url ( url , config , data = None , handlers = None ) : <EOL> """<STR_LIT>""" <EOL> debuglevel = <NUM_LIT:1> if config . debug else <NUM_LIT:0> <EOL> if config . cookie : <EOL> cj = config . cookie <EOL> else : <EOL> cj = cookiejar_ . CookieJar ( ) <EOL> cookie_handler = AccumulatingHTTPCookieProcessor ( cj ) <EOL> if not handlers : <EOL> handlers = [ ] <EOL> handlers . append ( cookie_handler ) <EOL> if config . debug : <EOL> http_handler = HTTPHandler_ ( debuglevel = debuglevel ) <EOL> https_handler = HTTPSContextHandler ( config . ssl_context , <EOL> debuglevel = debuglevel ) <EOL> handlers . extend ( [ http_handler , https_handler ] ) <EOL> if config . http_basicauth : <EOL> auth_handler = HTTPBasicAuthHandler_ ( HTTPPasswordMgrWithDefaultRealm_ ( ) ) <EOL> auth_handler . add_password ( realm = None , uri = url , <EOL> user = config . httpauth [ <NUM_LIT:0> ] , <EOL> passwd = config . httpauth [ <NUM_LIT:1> ] ) <EOL> handlers . append ( auth_handler ) <EOL> if not _should_use_proxy ( url , config . no_proxy ) : <EOL> handlers . append ( ProxyHandler_ ( { } ) ) <EOL> log . debug ( "<STR_LIT>" ) <EOL> elif config . proxies : <EOL> handlers . append ( ProxyHandler_ ( config . proxies ) ) <EOL> log . debug ( "<STR_LIT>" % config . proxies ) <EOL> opener = build_opener ( * handlers , ssl_context = config . ssl_context ) <EOL> headers = config . headers <EOL> if headers is None : <EOL> headers = { } <EOL> request = Request_ ( url , data , headers ) <EOL> return_code = <NUM_LIT:0> <EOL> return_message = '<STR_LIT>' <EOL> response = None <EOL> response = opener . open ( request ) <EOL> try : <EOL> response = opener . open ( request ) <EOL> return_message = response . msg <EOL> return_code = response . code <EOL> if log . isEnabledFor ( logging . DEBUG ) : <EOL> for index , cookie in enumerate ( cj ) : <EOL> log . debug ( "<STR_LIT>" , index , cookie ) <EOL> except HTTPError_ as exc : <EOL> return_code = exc . code <EOL> return_message = "<STR_LIT>" % exc . msg <EOL> if log . isEnabledFor ( logging . DEBUG ) : <EOL> log . debug ( "<STR_LIT>" , exc . code , exc . msg ) <EOL> except Exception as exc : <EOL> return_message = "<STR_LIT>" % exc . __str__ ( ) <EOL> if log . isEnabledFor ( logging . DEBUG ) : <EOL> import traceback <EOL> log . debug ( traceback . format_exc ( ) ) <EOL> return ( return_code , return_message , response ) <EOL> def _should_use_proxy ( url , no_proxy = None ) : <EOL> """<STR_LIT>""" <EOL> if no_proxy is None : <EOL> no_proxy_effective = os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> else : <EOL> no_proxy_effective = no_proxy <EOL> urlObj = urlparse_ . urlparse ( _url_as_string ( url ) ) <EOL> for np in [ h . strip ( ) for h in no_proxy_effective . split ( '<STR_LIT:U+002C>' ) ] : <EOL> if urlObj . hostname == np : <EOL> return False <EOL> return True <EOL> def _url_as_string ( url ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( url , Request_ ) : <EOL> return url . get_full_url ( ) <EOL> elif isinstance ( url , str ) : <EOL> return url <EOL> else : <EOL> raise TypeError ( "<STR_LIT>" % <EOL> ( str , Request_ ) ) <EOL> class Configuration ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , ssl_context , debug = False , proxies = None , no_proxy = None , <EOL> cookie = None , http_basicauth = None , headers = None ) : <EOL> """<STR_LIT>""" <EOL> self . ssl_context = ssl_context <EOL> self . debug = debug <EOL> self . proxies = proxies <EOL> self . no_proxy = no_proxy <EOL> self . cookie = cookie <EOL> self . http_basicauth = http_basicauth <EOL> self . headers = headers <EOL> def main ( ) : <EOL> '''<STR_LIT>''' <EOL> parser = OptionParser ( usage = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT:-c>" , "<STR_LIT>" , dest = "<STR_LIT>" , metavar = "<STR_LIT>" , <EOL> default = os . path . expanduser ( "<STR_LIT>" ) , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , metavar = "<STR_LIT>" , <EOL> default = None , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , <EOL> default = None , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , action = "<STR_LIT:store_true>" , dest = "<STR_LIT>" , <EOL> default = False , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , default = None , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , metavar = "<STR_LIT>" , <EOL> default = None , help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , action = "<STR_LIT:store_true>" , <EOL> dest = "<STR_LIT>" , default = False , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , <EOL> default = None , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , action = "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> metavar = "<STR_LIT>" , <EOL> help = "<STR_LIT>" ) <EOL> ( options , args ) = parser . parse_args ( ) <EOL> if len ( args ) != <NUM_LIT:1> : <EOL> parser . error ( "<STR_LIT>" ) <EOL> url = args [ <NUM_LIT:0> ] <EOL> if options . debug : <EOL> logging . getLogger ( ) . setLevel ( logging . DEBUG ) <EOL> if options . key_file and os . path . exists ( options . key_file ) : <EOL> key_file = options . key_file <EOL> else : <EOL> key_file = None <EOL> if options . cert_file and os . path . exists ( options . cert_file ) : <EOL> cert_file = options . cert_file <EOL> else : <EOL> cert_file = None <EOL> if options . ca_dir and os . path . exists ( options . ca_dir ) : <EOL> ca_dir = options . ca_dir <EOL> else : <EOL> ca_dir = None <EOL> verify_peer = not options . no_verify_peer <EOL> if options . data_file and os . path . exists ( options . data_file ) : <EOL> data_file = open ( options . data_file ) <EOL> data = data_file . read ( ) <EOL> data_file . close ( ) <EOL> else : <EOL> data = None <EOL> if options . basicauth : <EOL> http_basicauth = options . basicauth . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> else : <EOL> http_basicauth = None <EOL> headers = { } <EOL> if options . headers : <EOL> for h in options . headers : <EOL> key , val = h . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> headers [ key . strip ( ) ] = val . lstrip ( ) <EOL> ssl_context = ssl_context_util . make_ssl_context ( key_file , <EOL> cert_file , <EOL> None , <EOL> ca_dir , <EOL> verify_peer , <EOL> url ) <EOL> config = Configuration ( ssl_context , <EOL> options . debug , <EOL> http_basicauth = http_basicauth , <EOL> headers = headers ) <EOL> if options . output_file : <EOL> return_code , return_message = fetch_from_url_to_file ( <EOL> url , <EOL> config , <EOL> options . output_file , <EOL> data ) [ : <NUM_LIT:2> ] <EOL> raise SystemExit ( return_code , return_message ) <EOL> else : <EOL> data = fetch_from_url ( url , config ) <EOL> print ( data ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> logging . basicConfig ( ) <EOL> main ( ) </s>
