<s> import cgi <EOL> import getpass <EOL> import hashlib <EOL> import mimetypes <EOL> import os <EOL> import re <EOL> import shutil <EOL> import socket <EOL> import sys <EOL> import tempfile <EOL> from pip . backwardcompat import ( xmlrpclib , urllib , urllib2 , httplib , <EOL> urlparse , string_types , ssl ) <EOL> if ssl : <EOL> from pip . backwardcompat import match_hostname , CertificateError <EOL> from pip . exceptions import InstallationError , PipError , NoSSLError <EOL> from pip . util import ( splitext , rmtree , format_size , display_path , <EOL> backup_dir , ask_path_exists , unpack_file , <EOL> create_download_cache_folder , cache_download ) <EOL> from pip . vcs import vcs <EOL> from pip . log import logger <EOL> from pip . locations import default_cert_path <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> xmlrpclib_transport = xmlrpclib . Transport ( ) <EOL> def get_file_content ( url , comes_from = None ) : <EOL> """<STR_LIT>""" <EOL> match = _scheme_re . search ( url ) <EOL> if match : <EOL> scheme = match . group ( <NUM_LIT:1> ) . lower ( ) <EOL> if ( scheme == '<STR_LIT:file>' and comes_from <EOL> and comes_from . startswith ( '<STR_LIT:http>' ) ) : <EOL> raise InstallationError ( <EOL> '<STR_LIT>' <EOL> % ( comes_from , url ) ) <EOL> if scheme == '<STR_LIT:file>' : <EOL> path = url . split ( '<STR_LIT::>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> path = path . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> match = _url_slash_drive_re . match ( path ) <EOL> if match : <EOL> path = match . group ( <NUM_LIT:1> ) + '<STR_LIT::>' + path . split ( '<STR_LIT:|>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> path = urllib . unquote ( path ) <EOL> if path . startswith ( '<STR_LIT:/>' ) : <EOL> path = '<STR_LIT:/>' + path . lstrip ( '<STR_LIT:/>' ) <EOL> url = path <EOL> else : <EOL> resp = urlopen ( url ) <EOL> return geturl ( resp ) , resp . read ( ) <EOL> try : <EOL> f = open ( url ) <EOL> content = f . read ( ) <EOL> except IOError : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> raise InstallationError ( '<STR_LIT>' % str ( e ) ) <EOL> else : <EOL> f . close ( ) <EOL> return url , content <EOL> _scheme_re = re . compile ( r'<STR_LIT>' , re . I ) <EOL> _url_slash_drive_re = re . compile ( r'<STR_LIT>' , re . I ) <EOL> class VerifiedHTTPSConnection ( httplib . HTTPSConnection ) : <EOL> """<STR_LIT>""" <EOL> def connect ( self ) : <EOL> self . connection_kwargs = { } <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> self . connection_kwargs . update ( timeout = self . timeout ) <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> self . connection_kwargs . update ( source_address = self . source_address ) <EOL> sock = socket . create_connection ( ( self . host , self . port ) , ** self . connection_kwargs ) <EOL> if getattr ( self , '<STR_LIT>' , None ) : <EOL> self . sock = sock <EOL> self . _tunnel ( ) <EOL> cert_path = os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) or default_cert_path <EOL> self . sock = ssl . wrap_socket ( sock , <EOL> self . key_file , <EOL> self . cert_file , <EOL> cert_reqs = ssl . CERT_REQUIRED , <EOL> ca_certs = cert_path ) <EOL> try : <EOL> match_hostname ( self . sock . getpeercert ( ) , self . host ) <EOL> except CertificateError : <EOL> self . sock . shutdown ( socket . SHUT_RDWR ) <EOL> self . sock . close ( ) <EOL> raise <EOL> class VerifiedHTTPSHandler ( urllib2 . HTTPSHandler ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , connection_class = VerifiedHTTPSConnection ) : <EOL> self . specialized_conn_class = connection_class <EOL> urllib2 . HTTPSHandler . __init__ ( self ) <EOL> def https_open ( self , req ) : <EOL> return self . do_open ( self . specialized_conn_class , req ) <EOL> class URLOpener ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . passman = urllib2 . HTTPPasswordMgrWithDefaultRealm ( ) <EOL> def __call__ ( self , url ) : <EOL> """<STR_LIT>""" <EOL> url , username , password , scheme = self . extract_credentials ( url ) <EOL> if username is None : <EOL> try : <EOL> response = self . get_opener ( scheme = scheme ) . open ( url ) <EOL> except urllib2 . HTTPError : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> if e . code != <NUM_LIT> : <EOL> raise <EOL> response = self . get_response ( url ) <EOL> else : <EOL> response = self . get_response ( url , username , password ) <EOL> return response <EOL> def get_request ( self , url ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( url , string_types ) : <EOL> url = urllib2 . Request ( url , headers = { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> return url <EOL> def get_response ( self , url , username = None , password = None ) : <EOL> """<STR_LIT>""" <EOL> scheme , netloc , path , query , frag = urlparse . urlsplit ( url ) <EOL> req = self . get_request ( url ) <EOL> stored_username , stored_password = self . passman . find_user_password ( None , netloc ) <EOL> if stored_username is None : <EOL> if username is None and self . prompting : <EOL> username = urllib . quote ( raw_input ( '<STR_LIT>' % netloc ) ) <EOL> password = urllib . quote ( getpass . getpass ( '<STR_LIT>' ) ) <EOL> if username and password : <EOL> self . passman . add_password ( None , netloc , username , password ) <EOL> stored_username , stored_password = self . passman . find_user_password ( None , netloc ) <EOL> authhandler = urllib2 . HTTPBasicAuthHandler ( self . passman ) <EOL> opener = self . get_opener ( authhandler , scheme = scheme ) <EOL> return opener . open ( req ) <EOL> def get_opener ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if kwargs . get ( '<STR_LIT>' ) == '<STR_LIT>' : <EOL> if ssl : <EOL> https_handler = VerifiedHTTPSHandler ( ) <EOL> director = urllib2 . build_opener ( https_handler , * args ) <EOL> for handler in director . handlers : <EOL> if isinstance ( handler , urllib2 . HTTPHandler ) : <EOL> director . handlers . remove ( handler ) <EOL> return director <EOL> elif os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) == '<STR_LIT:1>' : <EOL> return urllib2 . build_opener ( * args ) <EOL> else : <EOL> raise NoSSLError ( ) <EOL> else : <EOL> return urllib2 . build_opener ( * args ) <EOL> def setup ( self , proxystr = '<STR_LIT>' , prompting = True ) : <EOL> """<STR_LIT>""" <EOL> self . prompting = prompting <EOL> proxy = self . get_proxy ( proxystr ) <EOL> if proxy : <EOL> proxy_support = urllib2 . ProxyHandler ( { "<STR_LIT:http>" : proxy , "<STR_LIT>" : proxy , "<STR_LIT>" : proxy } ) <EOL> opener = urllib2 . build_opener ( proxy_support , urllib2 . CacheFTPHandler ) <EOL> urllib2 . install_opener ( opener ) <EOL> def parse_credentials ( self , netloc ) : <EOL> if "<STR_LIT:@>" in netloc : <EOL> userinfo = netloc . rsplit ( "<STR_LIT:@>" , <NUM_LIT:1> ) [ <NUM_LIT:0> ] <EOL> if "<STR_LIT::>" in userinfo : <EOL> return userinfo . split ( "<STR_LIT::>" , <NUM_LIT:1> ) <EOL> return userinfo , None <EOL> return None , None <EOL> def extract_credentials ( self , url ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( url , urllib2 . Request ) : <EOL> result = urlparse . urlsplit ( url . get_full_url ( ) ) <EOL> else : <EOL> result = urlparse . urlsplit ( url ) <EOL> scheme , netloc , path , query , frag = result <EOL> username , password = self . parse_credentials ( netloc ) <EOL> if username is None : <EOL> return url , None , None , scheme <EOL> elif password is None and self . prompting : <EOL> netloc = netloc . replace ( '<STR_LIT>' % username , '<STR_LIT>' , <NUM_LIT:1> ) <EOL> prompt = '<STR_LIT>' % ( username , netloc ) <EOL> password = urllib . quote ( getpass . getpass ( prompt ) ) <EOL> else : <EOL> netloc = netloc . replace ( '<STR_LIT>' % ( username , password ) , '<STR_LIT>' , <NUM_LIT:1> ) <EOL> target_url = urlparse . urlunsplit ( ( scheme , netloc , path , query , frag ) ) <EOL> return target_url , username , password , scheme <EOL> def get_proxy ( self , proxystr = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> if not proxystr : <EOL> proxystr = os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if proxystr : <EOL> if '<STR_LIT:@>' in proxystr : <EOL> user_password , server_port = proxystr . split ( '<STR_LIT:@>' , <NUM_LIT:1> ) <EOL> if '<STR_LIT::>' in user_password : <EOL> user , password = user_password . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> else : <EOL> user = user_password <EOL> prompt = '<STR_LIT>' % ( user , server_port ) <EOL> password = urllib . quote ( getpass . getpass ( prompt ) ) <EOL> return '<STR_LIT>' % ( user , password , server_port ) <EOL> else : <EOL> return proxystr <EOL> else : <EOL> return None <EOL> urlopen = URLOpener ( ) <EOL> def is_url ( name ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT::>' not in name : <EOL> return False <EOL> scheme = name . split ( '<STR_LIT::>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] . lower ( ) <EOL> return scheme in [ '<STR_LIT:http>' , '<STR_LIT>' , '<STR_LIT:file>' , '<STR_LIT>' ] + vcs . all_schemes <EOL> def url_to_path ( url ) : <EOL> """<STR_LIT>""" <EOL> assert url . startswith ( '<STR_LIT>' ) , ( <EOL> "<STR_LIT>" % url ) <EOL> path = url [ len ( '<STR_LIT>' ) : ] . lstrip ( '<STR_LIT:/>' ) <EOL> path = urllib . unquote ( path ) <EOL> if _url_drive_re . match ( path ) : <EOL> path = path [ <NUM_LIT:0> ] + '<STR_LIT::>' + path [ <NUM_LIT:2> : ] <EOL> else : <EOL> path = '<STR_LIT:/>' + path <EOL> return path <EOL> _drive_re = re . compile ( '<STR_LIT>' , re . I ) <EOL> _url_drive_re = re . compile ( '<STR_LIT>' , re . I ) <EOL> def path_to_url ( path ) : <EOL> """<STR_LIT>""" <EOL> path = os . path . normcase ( os . path . abspath ( path ) ) <EOL> if _drive_re . match ( path ) : <EOL> path = path [ <NUM_LIT:0> ] + '<STR_LIT:|>' + path [ <NUM_LIT:2> : ] <EOL> url = urllib . quote ( path ) <EOL> url = url . replace ( os . path . sep , '<STR_LIT:/>' ) <EOL> url = url . lstrip ( '<STR_LIT:/>' ) <EOL> return '<STR_LIT>' + url <EOL> def path_to_url2 ( path ) : <EOL> """<STR_LIT>""" <EOL> path = os . path . normpath ( os . path . abspath ( path ) ) <EOL> drive , path = os . path . splitdrive ( path ) <EOL> filepath = path . split ( os . path . sep ) <EOL> url = '<STR_LIT:/>' . join ( [ urllib . quote ( part ) for part in filepath ] ) <EOL> if not drive : <EOL> url = url . lstrip ( '<STR_LIT:/>' ) <EOL> return '<STR_LIT>' + drive + url <EOL> def geturl ( urllib2_resp ) : <EOL> """<STR_LIT>""" <EOL> url = urllib2_resp . geturl ( ) <EOL> scheme , rest = url . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> if rest . startswith ( '<STR_LIT>' ) : <EOL> return url <EOL> else : <EOL> return '<STR_LIT>' % ( scheme , rest ) <EOL> def is_archive_file ( name ) : <EOL> """<STR_LIT>""" <EOL> archives = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> ext = splitext ( name ) [ <NUM_LIT:1> ] . lower ( ) <EOL> if ext in archives : <EOL> return True <EOL> return False <EOL> def unpack_vcs_link ( link , location , only_download = False ) : <EOL> vcs_backend = _get_used_vcs_backend ( link ) <EOL> if only_download : <EOL> vcs_backend . export ( location ) <EOL> else : <EOL> vcs_backend . unpack ( location ) <EOL> def unpack_file_url ( link , location ) : <EOL> source = url_to_path ( link . url ) <EOL> content_type = mimetypes . guess_type ( source ) [ <NUM_LIT:0> ] <EOL> if os . path . isdir ( source ) : <EOL> if os . path . isdir ( location ) : <EOL> rmtree ( location ) <EOL> shutil . copytree ( source , location ) <EOL> else : <EOL> unpack_file ( source , location , content_type , link ) <EOL> def _get_used_vcs_backend ( link ) : <EOL> for backend in vcs . backends : <EOL> if link . scheme in backend . schemes : <EOL> vcs_backend = backend ( link . url ) <EOL> return vcs_backend <EOL> def is_vcs_url ( link ) : <EOL> return bool ( _get_used_vcs_backend ( link ) ) <EOL> def is_file_url ( link ) : <EOL> return link . url . lower ( ) . startswith ( '<STR_LIT>' ) <EOL> def _check_hash ( download_hash , link ) : <EOL> if download_hash . digest_size != hashlib . new ( link . hash_name ) . digest_size : <EOL> logger . fatal ( "<STR_LIT>" <EOL> % ( download_hash . digest_size , link , link . hash_name ) ) <EOL> raise InstallationError ( '<STR_LIT>' % link ) <EOL> if download_hash . hexdigest ( ) != link . hash : <EOL> logger . fatal ( "<STR_LIT>" <EOL> % ( link , download_hash , link . hash ) ) <EOL> raise InstallationError ( '<STR_LIT>' % ( link . hash_name , link ) ) <EOL> def _get_hash_from_file ( target_file , link ) : <EOL> try : <EOL> download_hash = hashlib . new ( link . hash_name ) <EOL> except ( ValueError , TypeError ) : <EOL> logger . warn ( "<STR_LIT>" % ( link . hash_name , link ) ) <EOL> return None <EOL> fp = open ( target_file , '<STR_LIT:rb>' ) <EOL> while True : <EOL> chunk = fp . read ( <NUM_LIT> ) <EOL> if not chunk : <EOL> break <EOL> download_hash . update ( chunk ) <EOL> fp . close ( ) <EOL> return download_hash <EOL> def _download_url ( resp , link , temp_location ) : <EOL> fp = open ( temp_location , '<STR_LIT:wb>' ) <EOL> download_hash = None <EOL> if link . hash and link . hash_name : <EOL> try : <EOL> download_hash = hashlib . new ( link . hash_name ) <EOL> except ValueError : <EOL> logger . warn ( "<STR_LIT>" % ( link . hash_name , link ) ) <EOL> try : <EOL> total_length = int ( resp . info ( ) [ '<STR_LIT>' ] ) <EOL> except ( ValueError , KeyError , TypeError ) : <EOL> total_length = <NUM_LIT:0> <EOL> downloaded = <NUM_LIT:0> <EOL> show_progress = total_length > <NUM_LIT> * <NUM_LIT:1000> or not total_length <EOL> show_url = link . show_url <EOL> try : <EOL> if show_progress : <EOL> if total_length : <EOL> logger . start_progress ( '<STR_LIT>' % ( show_url , format_size ( total_length ) ) ) <EOL> else : <EOL> logger . start_progress ( '<STR_LIT>' % show_url ) <EOL> else : <EOL> logger . notify ( '<STR_LIT>' % show_url ) <EOL> logger . info ( '<STR_LIT>' % link ) <EOL> while True : <EOL> chunk = resp . read ( <NUM_LIT> ) <EOL> if not chunk : <EOL> break <EOL> downloaded += len ( chunk ) <EOL> if show_progress : <EOL> if not total_length : <EOL> logger . show_progress ( '<STR_LIT:%s>' % format_size ( downloaded ) ) <EOL> else : <EOL> logger . show_progress ( '<STR_LIT>' % ( <NUM_LIT:100> * downloaded / total_length , format_size ( downloaded ) ) ) <EOL> if download_hash is not None : <EOL> download_hash . update ( chunk ) <EOL> fp . write ( chunk ) <EOL> fp . close ( ) <EOL> finally : <EOL> if show_progress : <EOL> logger . end_progress ( '<STR_LIT>' % format_size ( downloaded ) ) <EOL> return download_hash <EOL> def _copy_file ( filename , location , content_type , link ) : <EOL> copy = True <EOL> download_location = os . path . join ( location , link . filename ) <EOL> if os . path . exists ( download_location ) : <EOL> response = ask_path_exists ( <EOL> '<STR_LIT>' % <EOL> display_path ( download_location ) , ( '<STR_LIT:i>' , '<STR_LIT:w>' , '<STR_LIT:b>' ) ) <EOL> if response == '<STR_LIT:i>' : <EOL> copy = False <EOL> elif response == '<STR_LIT:w>' : <EOL> logger . warn ( '<STR_LIT>' % display_path ( download_location ) ) <EOL> os . remove ( download_location ) <EOL> elif response == '<STR_LIT:b>' : <EOL> dest_file = backup_dir ( download_location ) <EOL> logger . warn ( '<STR_LIT>' <EOL> % ( display_path ( download_location ) , display_path ( dest_file ) ) ) <EOL> shutil . move ( download_location , dest_file ) <EOL> if copy : <EOL> shutil . copy ( filename , download_location ) <EOL> logger . indent -= <NUM_LIT:2> <EOL> logger . notify ( '<STR_LIT>' % display_path ( download_location ) ) <EOL> def unpack_http_url ( link , location , download_cache , download_dir = None ) : <EOL> temp_dir = tempfile . mkdtemp ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> target_url = link . url . split ( '<STR_LIT:#>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] <EOL> target_file = None <EOL> download_hash = None <EOL> if download_cache : <EOL> target_file = os . path . join ( download_cache , <EOL> urllib . quote ( target_url , '<STR_LIT>' ) ) <EOL> if not os . path . isdir ( download_cache ) : <EOL> create_download_cache_folder ( download_cache ) <EOL> already_downloaded = None <EOL> if download_dir : <EOL> already_downloaded = os . path . join ( download_dir , link . filename ) <EOL> if not os . path . exists ( already_downloaded ) : <EOL> already_downloaded = None <EOL> if ( target_file <EOL> and os . path . exists ( target_file ) <EOL> and os . path . exists ( target_file + '<STR_LIT>' ) ) : <EOL> fp = open ( target_file + '<STR_LIT>' ) <EOL> content_type = fp . read ( ) . strip ( ) <EOL> fp . close ( ) <EOL> if link . hash and link . hash_name : <EOL> download_hash = _get_hash_from_file ( target_file , link ) <EOL> temp_location = target_file <EOL> logger . notify ( '<STR_LIT>' % target_file ) <EOL> elif already_downloaded : <EOL> temp_location = already_downloaded <EOL> content_type = mimetypes . guess_type ( already_downloaded ) <EOL> if link . hash : <EOL> download_hash = _get_hash_from_file ( temp_location , link ) <EOL> logger . notify ( '<STR_LIT>' % already_downloaded ) <EOL> else : <EOL> resp = _get_response_from_url ( target_url , link ) <EOL> content_type = resp . info ( ) [ '<STR_LIT>' ] <EOL> filename = link . filename <EOL> content_disposition = resp . info ( ) . get ( '<STR_LIT>' ) <EOL> if content_disposition : <EOL> type , params = cgi . parse_header ( content_disposition ) <EOL> filename = params . get ( '<STR_LIT:filename>' ) or filename <EOL> ext = splitext ( filename ) [ <NUM_LIT:1> ] <EOL> if not ext : <EOL> ext = mimetypes . guess_extension ( content_type ) <EOL> if ext : <EOL> filename += ext <EOL> if not ext and link . url != geturl ( resp ) : <EOL> ext = os . path . splitext ( geturl ( resp ) ) [ <NUM_LIT:1> ] <EOL> if ext : <EOL> filename += ext <EOL> temp_location = os . path . join ( temp_dir , filename ) <EOL> download_hash = _download_url ( resp , link , temp_location ) <EOL> if link . hash and link . hash_name : <EOL> _check_hash ( download_hash , link ) <EOL> if download_dir and not already_downloaded : <EOL> _copy_file ( temp_location , download_dir , content_type , link ) <EOL> unpack_file ( temp_location , location , content_type , link ) <EOL> if target_file and target_file != temp_location : <EOL> cache_download ( target_file , temp_location , content_type ) <EOL> if target_file is None and not already_downloaded : <EOL> os . unlink ( temp_location ) <EOL> os . rmdir ( temp_dir ) <EOL> def _get_response_from_url ( target_url , link ) : <EOL> try : <EOL> resp = urlopen ( target_url ) <EOL> except urllib2 . HTTPError : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> logger . fatal ( "<STR_LIT>" % ( e . code , link ) ) <EOL> raise <EOL> except IOError : <EOL> e = sys . exc_info ( ) [ <NUM_LIT:1> ] <EOL> logger . fatal ( "<STR_LIT>" % ( e , link ) ) <EOL> raise <EOL> return resp <EOL> class Urllib2HeadRequest ( urllib2 . Request ) : <EOL> def get_method ( self ) : <EOL> return "<STR_LIT>" </s>
