<s> import codecs <EOL> from collections import deque <EOL> import contextlib <EOL> import csv <EOL> from glob import iglob as std_iglob <EOL> import io <EOL> import json <EOL> import logging <EOL> import os <EOL> import py_compile <EOL> import re <EOL> import shutil <EOL> import socket <EOL> import ssl <EOL> import subprocess <EOL> import sys <EOL> import tarfile <EOL> import tempfile <EOL> import textwrap <EOL> try : <EOL> import threading <EOL> except ImportError : <EOL> import dummy_threading as threading <EOL> import time <EOL> from . import DistlibException <EOL> from . compat import ( string_types , text_type , shutil , raw_input , StringIO , <EOL> cache_from_source , urlopen , urljoin , httplib , xmlrpclib , <EOL> splittype , HTTPHandler , HTTPSHandler as BaseHTTPSHandler , <EOL> BaseConfigurator , valid_ident , Container , configparser , <EOL> URLError , match_hostname , CertificateError , ZipFile ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> COMMA = r'<STR_LIT>' <EOL> COMMA_RE = re . compile ( COMMA ) <EOL> IDENT = r'<STR_LIT>' <EOL> EXTRA_IDENT = r'<STR_LIT>' + IDENT + '<STR_LIT:)>' <EOL> VERSPEC = IDENT + r'<STR_LIT>' <EOL> RELOP = '<STR_LIT>' <EOL> BARE_CONSTRAINTS = ( '<STR_LIT:(>' + RELOP + r'<STR_LIT>' + VERSPEC + '<STR_LIT>' + COMMA + '<STR_LIT:(>' + <EOL> RELOP + r'<STR_LIT>' + VERSPEC + '<STR_LIT>' ) <EOL> DIRECT_REF = '<STR_LIT>' <EOL> CONSTRAINTS = ( r'<STR_LIT>' + BARE_CONSTRAINTS + '<STR_LIT:|>' + DIRECT_REF + <EOL> r'<STR_LIT>' + BARE_CONSTRAINTS + '<STR_LIT>' ) <EOL> EXTRA_LIST = EXTRA_IDENT + '<STR_LIT:(>' + COMMA + EXTRA_IDENT + '<STR_LIT>' <EOL> EXTRAS = r'<STR_LIT>' + EXTRA_LIST + r'<STR_LIT>' <EOL> REQUIREMENT = ( '<STR_LIT>' + IDENT + r'<STR_LIT>' + EXTRAS + r'<STR_LIT>' + <EOL> CONSTRAINTS + '<STR_LIT>' ) <EOL> REQUIREMENT_RE = re . compile ( REQUIREMENT ) <EOL> RELOP_IDENT = '<STR_LIT>' + RELOP + r'<STR_LIT>' + VERSPEC + '<STR_LIT:)>' <EOL> RELOP_IDENT_RE = re . compile ( RELOP_IDENT ) <EOL> def parse_requirement ( s ) : <EOL> def get_constraint ( m ) : <EOL> d = m . groupdict ( ) <EOL> return d [ '<STR_LIT>' ] , d [ '<STR_LIT>' ] <EOL> result = None <EOL> m = REQUIREMENT_RE . match ( s ) <EOL> if m : <EOL> d = m . groupdict ( ) <EOL> name = d [ '<STR_LIT>' ] <EOL> cons = d [ '<STR_LIT>' ] or d [ '<STR_LIT>' ] <EOL> if not d [ '<STR_LIT>' ] : <EOL> url = None <EOL> else : <EOL> cons = None <EOL> url = d [ '<STR_LIT>' ] . strip ( ) <EOL> if not cons : <EOL> cons = None <EOL> constr = '<STR_LIT>' <EOL> rs = d [ '<STR_LIT>' ] <EOL> else : <EOL> if cons [ <NUM_LIT:0> ] not in '<STR_LIT>' : <EOL> cons = '<STR_LIT>' + cons <EOL> iterator = RELOP_IDENT_RE . finditer ( cons ) <EOL> cons = [ get_constraint ( m ) for m in iterator ] <EOL> rs = '<STR_LIT>' % ( name , '<STR_LIT:U+002CU+0020>' . join ( [ '<STR_LIT>' % con for con in cons ] ) ) <EOL> if not d [ '<STR_LIT>' ] : <EOL> extras = None <EOL> else : <EOL> extras = COMMA_RE . split ( d [ '<STR_LIT>' ] ) <EOL> result = Container ( name = name , constraints = cons , extras = extras , <EOL> requirement = rs , source = s , url = url ) <EOL> return result <EOL> def get_resources_dests ( resources_root , rules ) : <EOL> """<STR_LIT>""" <EOL> def get_rel_path ( base , path ) : <EOL> base = base . replace ( os . path . sep , '<STR_LIT:/>' ) <EOL> path = path . replace ( os . path . sep , '<STR_LIT:/>' ) <EOL> assert path . startswith ( base ) <EOL> return path [ len ( base ) : ] . lstrip ( '<STR_LIT:/>' ) <EOL> destinations = { } <EOL> for base , suffix , dest in rules : <EOL> prefix = os . path . join ( resources_root , base ) <EOL> for abs_base in iglob ( prefix ) : <EOL> abs_glob = os . path . join ( abs_base , suffix ) <EOL> for abs_path in iglob ( abs_glob ) : <EOL> resource_file = get_rel_path ( resources_root , abs_path ) <EOL> if dest is None : <EOL> destinations . pop ( resource_file , None ) <EOL> else : <EOL> rel_path = get_rel_path ( abs_base , abs_path ) <EOL> rel_dest = dest . replace ( os . path . sep , '<STR_LIT:/>' ) . rstrip ( '<STR_LIT:/>' ) <EOL> destinations [ resource_file ] = rel_dest + '<STR_LIT:/>' + rel_path <EOL> return destinations <EOL> def in_venv ( ) : <EOL> if hasattr ( sys , '<STR_LIT>' ) : <EOL> result = True <EOL> else : <EOL> result = sys . prefix != getattr ( sys , '<STR_LIT>' , sys . prefix ) <EOL> return result <EOL> def get_executable ( ) : <EOL> return os . path . normcase ( sys . executable ) <EOL> def proceed ( prompt , allowed_chars , error_prompt = None , default = None ) : <EOL> p = prompt <EOL> while True : <EOL> s = raw_input ( p ) <EOL> p = prompt <EOL> if not s and default : <EOL> s = default <EOL> if s : <EOL> c = s [ <NUM_LIT:0> ] . lower ( ) <EOL> if c in allowed_chars : <EOL> break <EOL> if error_prompt : <EOL> p = '<STR_LIT>' % ( c , error_prompt , prompt ) <EOL> return c <EOL> def extract_by_key ( d , keys ) : <EOL> if isinstance ( keys , string_types ) : <EOL> keys = keys . split ( ) <EOL> result = { } <EOL> for key in keys : <EOL> if key in d : <EOL> result [ key ] = d [ key ] <EOL> return result <EOL> def read_exports ( stream ) : <EOL> if sys . version_info [ <NUM_LIT:0> ] >= <NUM_LIT:3> : <EOL> stream = codecs . getreader ( '<STR_LIT:utf-8>' ) ( stream ) <EOL> data = stream . read ( ) <EOL> stream = StringIO ( data ) <EOL> try : <EOL> jdata = json . load ( stream ) <EOL> result = jdata [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> for group , entries in result . items ( ) : <EOL> for k , v in entries . items ( ) : <EOL> s = '<STR_LIT>' % ( k , v ) <EOL> entry = get_export_entry ( s ) <EOL> assert entry is not None <EOL> entries [ k ] = entry <EOL> return result <EOL> except Exception : <EOL> stream . seek ( <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> def read_stream ( cp , stream ) : <EOL> if hasattr ( cp , '<STR_LIT>' ) : <EOL> cp . read_file ( stream ) <EOL> else : <EOL> cp . readfp ( stream ) <EOL> cp = configparser . ConfigParser ( ) <EOL> try : <EOL> read_stream ( cp , stream ) <EOL> except configparser . MissingSectionHeaderError : <EOL> stream . close ( ) <EOL> data = textwrap . dedent ( data ) <EOL> stream = StringIO ( data ) <EOL> read_stream ( cp , stream ) <EOL> result = { } <EOL> for key in cp . sections ( ) : <EOL> result [ key ] = entries = { } <EOL> for name , value in cp . items ( key ) : <EOL> s = '<STR_LIT>' % ( name , value ) <EOL> entry = get_export_entry ( s ) <EOL> assert entry is not None <EOL> entries [ name ] = entry <EOL> return result <EOL> def write_exports ( exports , stream ) : <EOL> if sys . version_info [ <NUM_LIT:0> ] >= <NUM_LIT:3> : <EOL> stream = codecs . getwriter ( '<STR_LIT:utf-8>' ) ( stream ) <EOL> cp = configparser . ConfigParser ( ) <EOL> for k , v in exports . items ( ) : <EOL> cp . add_section ( k ) <EOL> for entry in v . values ( ) : <EOL> if entry . suffix is None : <EOL> s = entry . prefix <EOL> else : <EOL> s = '<STR_LIT>' % ( entry . prefix , entry . suffix ) <EOL> if entry . flags : <EOL> s = '<STR_LIT>' % ( s , '<STR_LIT:U+002CU+0020>' . join ( entry . flags ) ) <EOL> cp . set ( k , entry . name , s ) <EOL> cp . write ( stream ) <EOL> @ contextlib . contextmanager <EOL> def tempdir ( ) : <EOL> td = tempfile . mkdtemp ( ) <EOL> try : <EOL> yield td <EOL> finally : <EOL> shutil . rmtree ( td ) <EOL> @ contextlib . contextmanager <EOL> def chdir ( d ) : <EOL> cwd = os . getcwd ( ) <EOL> try : <EOL> os . chdir ( d ) <EOL> yield <EOL> finally : <EOL> os . chdir ( cwd ) <EOL> @ contextlib . contextmanager <EOL> def socket_timeout ( seconds = <NUM_LIT:15> ) : <EOL> cto = socket . getdefaulttimeout ( ) <EOL> try : <EOL> socket . setdefaulttimeout ( seconds ) <EOL> yield <EOL> finally : <EOL> socket . setdefaulttimeout ( cto ) <EOL> class cached_property ( object ) : <EOL> def __init__ ( self , func ) : <EOL> self . func = func <EOL> def __get__ ( self , obj , cls = None ) : <EOL> if obj is None : <EOL> return self <EOL> value = self . func ( obj ) <EOL> object . __setattr__ ( obj , self . func . __name__ , value ) <EOL> return value <EOL> def convert_path ( pathname ) : <EOL> """<STR_LIT>""" <EOL> if os . sep == '<STR_LIT:/>' : <EOL> return pathname <EOL> if not pathname : <EOL> return pathname <EOL> if pathname [ <NUM_LIT:0> ] == '<STR_LIT:/>' : <EOL> raise ValueError ( "<STR_LIT>" % pathname ) <EOL> if pathname [ - <NUM_LIT:1> ] == '<STR_LIT:/>' : <EOL> raise ValueError ( "<STR_LIT>" % pathname ) <EOL> paths = pathname . split ( '<STR_LIT:/>' ) <EOL> while os . curdir in paths : <EOL> paths . remove ( os . curdir ) <EOL> if not paths : <EOL> return os . curdir <EOL> return os . path . join ( * paths ) <EOL> class FileOperator ( object ) : <EOL> def __init__ ( self , dry_run = False ) : <EOL> self . dry_run = dry_run <EOL> self . ensured = set ( ) <EOL> self . _init_record ( ) <EOL> def _init_record ( self ) : <EOL> self . record = False <EOL> self . files_written = set ( ) <EOL> self . dirs_created = set ( ) <EOL> def record_as_written ( self , path ) : <EOL> if self . record : <EOL> self . files_written . add ( path ) <EOL> def newer ( self , source , target ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . exists ( source ) : <EOL> raise DistlibException ( "<STR_LIT>" % <EOL> os . path . abspath ( source ) ) <EOL> if not os . path . exists ( target ) : <EOL> return True <EOL> return os . stat ( source ) . st_mtime > os . stat ( target ) . st_mtime <EOL> def copy_file ( self , infile , outfile , check = True ) : <EOL> """<STR_LIT>""" <EOL> self . ensure_dir ( os . path . dirname ( outfile ) ) <EOL> logger . info ( '<STR_LIT>' , infile , outfile ) <EOL> if not self . dry_run : <EOL> msg = None <EOL> if check : <EOL> if os . path . islink ( outfile ) : <EOL> msg = '<STR_LIT>' % outfile <EOL> elif os . path . exists ( outfile ) and not os . path . isfile ( outfile ) : <EOL> msg = '<STR_LIT>' % outfile <EOL> if msg : <EOL> raise ValueError ( msg + '<STR_LIT>' ) <EOL> shutil . copyfile ( infile , outfile ) <EOL> self . record_as_written ( outfile ) <EOL> def copy_stream ( self , instream , outfile , encoding = None ) : <EOL> assert not os . path . isdir ( outfile ) <EOL> self . ensure_dir ( os . path . dirname ( outfile ) ) <EOL> logger . info ( '<STR_LIT>' , instream , outfile ) <EOL> if not self . dry_run : <EOL> if encoding is None : <EOL> outstream = open ( outfile , '<STR_LIT:wb>' ) <EOL> else : <EOL> outstream = codecs . open ( outfile , '<STR_LIT:w>' , encoding = encoding ) <EOL> try : <EOL> shutil . copyfileobj ( instream , outstream ) <EOL> finally : <EOL> outstream . close ( ) <EOL> self . record_as_written ( outfile ) <EOL> def write_binary_file ( self , path , data ) : <EOL> self . ensure_dir ( os . path . dirname ( path ) ) <EOL> if not self . dry_run : <EOL> with open ( path , '<STR_LIT:wb>' ) as f : <EOL> f . write ( data ) <EOL> self . record_as_written ( path ) <EOL> def write_text_file ( self , path , data , encoding ) : <EOL> self . ensure_dir ( os . path . dirname ( path ) ) <EOL> if not self . dry_run : <EOL> with open ( path , '<STR_LIT:wb>' ) as f : <EOL> f . write ( data . encode ( encoding ) ) <EOL> self . record_as_written ( path ) <EOL> def set_mode ( self , bits , mask , files ) : <EOL> if os . name == '<STR_LIT>' or ( os . name == '<STR_LIT>' and os . _name == '<STR_LIT>' ) : <EOL> for f in files : <EOL> if self . dry_run : <EOL> logger . info ( "<STR_LIT>" , f ) <EOL> else : <EOL> mode = ( os . stat ( f ) . st_mode | bits ) & mask <EOL> logger . info ( "<STR_LIT>" , f , mode ) <EOL> os . chmod ( f , mode ) <EOL> set_executable_mode = lambda s , f : s . set_mode ( <NUM_LIT> , <NUM_LIT> , f ) <EOL> def ensure_dir ( self , path ) : <EOL> path = os . path . abspath ( path ) <EOL> if path not in self . ensured and not os . path . exists ( path ) : <EOL> self . ensured . add ( path ) <EOL> d , f = os . path . split ( path ) <EOL> self . ensure_dir ( d ) <EOL> logger . info ( '<STR_LIT>' % path ) <EOL> if not self . dry_run : <EOL> os . mkdir ( path ) <EOL> if self . record : <EOL> self . dirs_created . add ( path ) <EOL> def byte_compile ( self , path , optimize = False , force = False , prefix = None ) : <EOL> dpath = cache_from_source ( path , not optimize ) <EOL> logger . info ( '<STR_LIT>' , path , dpath ) <EOL> if not self . dry_run : <EOL> if force or self . newer ( path , dpath ) : <EOL> if not prefix : <EOL> diagpath = None <EOL> else : <EOL> assert path . startswith ( prefix ) <EOL> diagpath = path [ len ( prefix ) : ] <EOL> py_compile . compile ( path , dpath , diagpath , True ) <EOL> self . record_as_written ( dpath ) <EOL> return dpath <EOL> def ensure_removed ( self , path ) : <EOL> if os . path . exists ( path ) : <EOL> if os . path . isdir ( path ) and not os . path . islink ( path ) : <EOL> logger . debug ( '<STR_LIT>' , path ) <EOL> if not self . dry_run : <EOL> shutil . rmtree ( path ) <EOL> if self . record : <EOL> if path in self . dirs_created : <EOL> self . dirs_created . remove ( path ) <EOL> else : <EOL> if os . path . islink ( path ) : <EOL> s = '<STR_LIT>' <EOL> else : <EOL> s = '<STR_LIT:file>' <EOL> logger . debug ( '<STR_LIT>' , s , path ) <EOL> if not self . dry_run : <EOL> os . remove ( path ) <EOL> if self . record : <EOL> if path in self . files_written : <EOL> self . files_written . remove ( path ) <EOL> def is_writable ( self , path ) : <EOL> result = False <EOL> while not result : <EOL> if os . path . exists ( path ) : <EOL> result = os . access ( path , os . W_OK ) <EOL> break <EOL> parent = os . path . dirname ( path ) <EOL> if parent == path : <EOL> break <EOL> path = parent <EOL> return result <EOL> def commit ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . record <EOL> result = self . files_written , self . dirs_created <EOL> self . _init_record ( ) <EOL> return result <EOL> def rollback ( self ) : <EOL> if not self . dry_run : <EOL> for f in list ( self . files_written ) : <EOL> if os . path . exists ( f ) : <EOL> os . remove ( f ) <EOL> dirs = sorted ( self . dirs_created , reverse = True ) <EOL> for d in dirs : <EOL> flist = os . listdir ( d ) <EOL> if flist : <EOL> assert flist == [ '<STR_LIT>' ] <EOL> sd = os . path . join ( d , flist [ <NUM_LIT:0> ] ) <EOL> os . rmdir ( sd ) <EOL> os . rmdir ( d ) <EOL> self . _init_record ( ) <EOL> def resolve ( module_name , dotted_path ) : <EOL> if module_name in sys . modules : <EOL> mod = sys . modules [ module_name ] <EOL> else : <EOL> mod = __import__ ( module_name ) <EOL> if dotted_path is None : <EOL> result = mod <EOL> else : <EOL> parts = dotted_path . split ( '<STR_LIT:.>' ) <EOL> result = getattr ( mod , parts . pop ( <NUM_LIT:0> ) ) <EOL> for p in parts : <EOL> result = getattr ( result , p ) <EOL> return result <EOL> class ExportEntry ( object ) : <EOL> def __init__ ( self , name , prefix , suffix , flags ) : <EOL> self . name = name <EOL> self . prefix = prefix <EOL> self . suffix = suffix <EOL> self . flags = flags <EOL> @ cached_property <EOL> def value ( self ) : <EOL> return resolve ( self . prefix , self . suffix ) <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . name , self . prefix , <EOL> self . suffix , self . flags ) <EOL> def __eq__ ( self , other ) : <EOL> if not isinstance ( other , ExportEntry ) : <EOL> result = False <EOL> else : <EOL> result = ( self . name == other . name and <EOL> self . prefix == other . prefix and <EOL> self . suffix == other . suffix and <EOL> self . flags == other . flags ) <EOL> return result <EOL> __hash__ = object . __hash__ <EOL> ENTRY_RE = re . compile ( r'''<STR_LIT>''' , re . VERBOSE ) <EOL> def get_export_entry ( specification ) : <EOL> m = ENTRY_RE . search ( specification ) <EOL> if not m : <EOL> result = None <EOL> if '<STR_LIT:[>' in specification or '<STR_LIT:]>' in specification : <EOL> raise DistlibException ( '<STR_LIT>' <EOL> '<STR_LIT>' % specification ) <EOL> else : <EOL> d = m . groupdict ( ) <EOL> name = d [ '<STR_LIT:name>' ] <EOL> path = d [ '<STR_LIT>' ] <EOL> colons = path . count ( '<STR_LIT::>' ) <EOL> if colons == <NUM_LIT:0> : <EOL> prefix , suffix = path , None <EOL> else : <EOL> if colons != <NUM_LIT:1> : <EOL> raise DistlibException ( '<STR_LIT>' <EOL> '<STR_LIT>' % specification ) <EOL> prefix , suffix = path . split ( '<STR_LIT::>' ) <EOL> flags = d [ '<STR_LIT>' ] <EOL> if flags is None : <EOL> if '<STR_LIT:[>' in specification or '<STR_LIT:]>' in specification : <EOL> raise DistlibException ( '<STR_LIT>' <EOL> '<STR_LIT>' % specification ) <EOL> flags = [ ] <EOL> else : <EOL> flags = [ f . strip ( ) for f in flags . split ( '<STR_LIT:U+002C>' ) ] <EOL> result = ExportEntry ( name , prefix , suffix , flags ) <EOL> return result <EOL> def get_cache_base ( suffix = None ) : <EOL> """<STR_LIT>""" <EOL> if suffix is None : <EOL> suffix = '<STR_LIT>' <EOL> if os . name == '<STR_LIT>' and '<STR_LIT>' in os . environ : <EOL> result = os . path . expandvars ( '<STR_LIT>' ) <EOL> else : <EOL> result = os . path . expanduser ( '<STR_LIT>' ) <EOL> if os . path . isdir ( result ) : <EOL> usable = os . access ( result , os . W_OK ) <EOL> if not usable : <EOL> logger . warning ( '<STR_LIT>' , result ) <EOL> else : <EOL> try : <EOL> os . makedirs ( result ) <EOL> usable = True <EOL> except OSError : <EOL> logger . warning ( '<STR_LIT>' , result , exc_info = True ) <EOL> usable = False <EOL> if not usable : <EOL> result = tempfile . mkdtemp ( ) <EOL> logger . warning ( '<STR_LIT>' , result ) <EOL> return os . path . join ( result , suffix ) <EOL> def path_to_cache_dir ( path ) : <EOL> """<STR_LIT>""" <EOL> d , p = os . path . splitdrive ( os . path . abspath ( path ) ) <EOL> if d : <EOL> d = d . replace ( '<STR_LIT::>' , '<STR_LIT>' ) <EOL> p = p . replace ( os . sep , '<STR_LIT>' ) <EOL> return d + p + '<STR_LIT>' <EOL> def ensure_slash ( s ) : <EOL> if not s . endswith ( '<STR_LIT:/>' ) : <EOL> return s + '<STR_LIT:/>' <EOL> return s <EOL> def parse_credentials ( netloc ) : <EOL> username = password = None <EOL> if '<STR_LIT:@>' in netloc : <EOL> prefix , netloc = netloc . split ( '<STR_LIT:@>' , <NUM_LIT:1> ) <EOL> if '<STR_LIT::>' not in prefix : <EOL> username = prefix <EOL> else : <EOL> username , password = prefix . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> return username , password , netloc <EOL> def get_process_umask ( ) : <EOL> result = os . umask ( <NUM_LIT> ) <EOL> os . umask ( result ) <EOL> return result <EOL> def is_string_sequence ( seq ) : <EOL> result = True <EOL> i = None <EOL> for i , s in enumerate ( seq ) : <EOL> if not isinstance ( s , string_types ) : <EOL> result = False <EOL> break <EOL> assert i is not None <EOL> return result <EOL> PROJECT_NAME_AND_VERSION = re . compile ( '<STR_LIT>' <EOL> '<STR_LIT>' , re . I ) <EOL> PYTHON_VERSION = re . compile ( r'<STR_LIT>' ) <EOL> def split_filename ( filename , project_name = None ) : <EOL> """<STR_LIT>""" <EOL> result = None <EOL> pyver = None <EOL> m = PYTHON_VERSION . search ( filename ) <EOL> if m : <EOL> pyver = m . group ( <NUM_LIT:1> ) <EOL> filename = filename [ : m . start ( ) ] <EOL> if project_name and len ( filename ) > len ( project_name ) + <NUM_LIT:1> : <EOL> m = re . match ( re . escape ( project_name ) + r'<STR_LIT>' , filename ) <EOL> if m : <EOL> n = m . end ( ) <EOL> result = filename [ : n ] , filename [ n + <NUM_LIT:1> : ] , pyver <EOL> if result is None : <EOL> m = PROJECT_NAME_AND_VERSION . match ( filename ) <EOL> if m : <EOL> result = m . group ( <NUM_LIT:1> ) , m . group ( <NUM_LIT:3> ) , pyver <EOL> return result <EOL> NAME_VERSION_RE = re . compile ( r'<STR_LIT>' <EOL> r'<STR_LIT>' ) <EOL> def parse_name_and_version ( p ) : <EOL> """<STR_LIT>""" <EOL> m = NAME_VERSION_RE . match ( p ) <EOL> if not m : <EOL> raise DistlibException ( '<STR_LIT>' % p ) <EOL> d = m . groupdict ( ) <EOL> return d [ '<STR_LIT:name>' ] . strip ( ) . lower ( ) , d [ '<STR_LIT>' ] <EOL> def get_extras ( requested , available ) : <EOL> result = set ( ) <EOL> requested = set ( requested or [ ] ) <EOL> available = set ( available or [ ] ) <EOL> if '<STR_LIT:*>' in requested : <EOL> requested . remove ( '<STR_LIT:*>' ) <EOL> result |= available <EOL> for r in requested : <EOL> if r == '<STR_LIT:->' : <EOL> result . add ( r ) <EOL> elif r . startswith ( '<STR_LIT:->' ) : <EOL> unwanted = r [ <NUM_LIT:1> : ] <EOL> if unwanted not in available : <EOL> logger . warning ( '<STR_LIT>' % unwanted ) <EOL> if unwanted in result : <EOL> result . remove ( unwanted ) <EOL> else : <EOL> if r not in available : <EOL> logger . warning ( '<STR_LIT>' % r ) <EOL> result . add ( r ) <EOL> return result <EOL> def _get_external_data ( url ) : <EOL> result = { } <EOL> try : <EOL> resp = urlopen ( url ) <EOL> headers = resp . info ( ) <EOL> if headers . get ( '<STR_LIT:Content-Type>' ) != '<STR_LIT:application/json>' : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> else : <EOL> reader = codecs . getreader ( '<STR_LIT:utf-8>' ) ( resp ) <EOL> result = json . load ( reader ) <EOL> except Exception as e : <EOL> logger . exception ( '<STR_LIT>' , url , e ) <EOL> return result <EOL> _external_data_base_url = '<STR_LIT>' <EOL> def get_project_data ( name ) : <EOL> url = '<STR_LIT>' % ( name [ <NUM_LIT:0> ] . upper ( ) , name ) <EOL> url = urljoin ( _external_data_base_url , url ) <EOL> result = _get_external_data ( url ) <EOL> return result <EOL> def get_package_data ( name , version ) : <EOL> url = '<STR_LIT>' % ( name [ <NUM_LIT:0> ] . upper ( ) , name , version ) <EOL> url = urljoin ( _external_data_base_url , url ) <EOL> return _get_external_data ( url ) <EOL> class Cache ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , base ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( base ) : <EOL> os . makedirs ( base ) <EOL> if ( os . stat ( base ) . st_mode & <NUM_LIT> ) != <NUM_LIT:0> : <EOL> logger . warning ( '<STR_LIT>' , base ) <EOL> self . base = os . path . abspath ( os . path . normpath ( base ) ) <EOL> def prefix_to_dir ( self , prefix ) : <EOL> """<STR_LIT>""" <EOL> return path_to_cache_dir ( prefix ) <EOL> def clear ( self ) : <EOL> """<STR_LIT>""" <EOL> not_removed = [ ] <EOL> for fn in os . listdir ( self . base ) : <EOL> fn = os . path . join ( self . base , fn ) <EOL> try : <EOL> if os . path . islink ( fn ) or os . path . isfile ( fn ) : <EOL> os . remove ( fn ) <EOL> elif os . path . isdir ( fn ) : <EOL> shutil . rmtree ( fn ) <EOL> except Exception : <EOL> not_removed . append ( fn ) <EOL> return not_removed <EOL> class EventMixin ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _subscribers = { } <EOL> def add ( self , event , subscriber , append = True ) : <EOL> """<STR_LIT>""" <EOL> subs = self . _subscribers <EOL> if event not in subs : <EOL> subs [ event ] = deque ( [ subscriber ] ) <EOL> else : <EOL> sq = subs [ event ] <EOL> if append : <EOL> sq . append ( subscriber ) <EOL> else : <EOL> sq . appendleft ( subscriber ) <EOL> def remove ( self , event , subscriber ) : <EOL> """<STR_LIT>""" <EOL> subs = self . _subscribers <EOL> if event not in subs : <EOL> raise ValueError ( '<STR_LIT>' % event ) <EOL> subs [ event ] . remove ( subscriber ) <EOL> def get_subscribers ( self , event ) : <EOL> """<STR_LIT>""" <EOL> return iter ( self . _subscribers . get ( event , ( ) ) ) <EOL> def publish ( self , event , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> result = [ ] <EOL> for subscriber in self . get_subscribers ( event ) : <EOL> try : <EOL> value = subscriber ( event , * args , ** kwargs ) <EOL> except Exception : <EOL> logger . exception ( '<STR_LIT>' ) <EOL> value = None <EOL> result . append ( value ) <EOL> logger . debug ( '<STR_LIT>' , <EOL> event , args , kwargs , result ) <EOL> return result <EOL> class Sequencer ( object ) : <EOL> def __init__ ( self ) : <EOL> self . _preds = { } <EOL> self . _succs = { } <EOL> self . _nodes = set ( ) <EOL> def add_node ( self , node ) : <EOL> self . _nodes . add ( node ) <EOL> def remove_node ( self , node , edges = False ) : <EOL> if node in self . _nodes : <EOL> self . _nodes . remove ( node ) <EOL> if edges : <EOL> for p in set ( self . _preds . get ( node , ( ) ) ) : <EOL> self . remove ( p , node ) <EOL> for s in set ( self . _succs . get ( node , ( ) ) ) : <EOL> self . remove ( node , s ) <EOL> for k , v in list ( self . _preds . items ( ) ) : <EOL> if not v : <EOL> del self . _preds [ k ] <EOL> for k , v in list ( self . _succs . items ( ) ) : <EOL> if not v : <EOL> del self . _succs [ k ] <EOL> def add ( self , pred , succ ) : <EOL> assert pred != succ <EOL> self . _preds . setdefault ( succ , set ( ) ) . add ( pred ) <EOL> self . _succs . setdefault ( pred , set ( ) ) . add ( succ ) <EOL> def remove ( self , pred , succ ) : <EOL> assert pred != succ <EOL> try : <EOL> preds = self . _preds [ succ ] <EOL> succs = self . _succs [ pred ] <EOL> except KeyError : <EOL> raise ValueError ( '<STR_LIT>' % succ ) <EOL> try : <EOL> preds . remove ( pred ) <EOL> succs . remove ( succ ) <EOL> except KeyError : <EOL> raise ValueError ( '<STR_LIT>' % ( succ , pred ) ) <EOL> def is_step ( self , step ) : <EOL> return ( step in self . _preds or step in self . _succs or <EOL> step in self . _nodes ) <EOL> def get_steps ( self , final ) : <EOL> if not self . is_step ( final ) : <EOL> raise ValueError ( '<STR_LIT>' % final ) <EOL> result = [ ] <EOL> todo = [ ] <EOL> seen = set ( ) <EOL> todo . append ( final ) <EOL> while todo : <EOL> step = todo . pop ( <NUM_LIT:0> ) <EOL> if step in seen : <EOL> if step != final : <EOL> result . remove ( step ) <EOL> result . append ( step ) <EOL> else : <EOL> seen . add ( step ) <EOL> result . append ( step ) <EOL> preds = self . _preds . get ( step , ( ) ) <EOL> todo . extend ( preds ) <EOL> return reversed ( result ) <EOL> @ property <EOL> def strong_connections ( self ) : <EOL> index_counter = [ <NUM_LIT:0> ] <EOL> stack = [ ] <EOL> lowlinks = { } <EOL> index = { } <EOL> result = [ ] <EOL> graph = self . _succs <EOL> def strongconnect ( node ) : <EOL> index [ node ] = index_counter [ <NUM_LIT:0> ] <EOL> lowlinks [ node ] = index_counter [ <NUM_LIT:0> ] <EOL> index_counter [ <NUM_LIT:0> ] += <NUM_LIT:1> <EOL> stack . append ( node ) <EOL> try : <EOL> successors = graph [ node ] <EOL> except Exception : <EOL> successors = [ ] <EOL> for successor in successors : <EOL> if successor not in lowlinks : <EOL> strongconnect ( successor ) <EOL> lowlinks [ node ] = min ( lowlinks [ node ] , lowlinks [ successor ] ) <EOL> elif successor in stack : <EOL> lowlinks [ node ] = min ( lowlinks [ node ] , index [ successor ] ) <EOL> if lowlinks [ node ] == index [ node ] : <EOL> connected_component = [ ] <EOL> while True : <EOL> successor = stack . pop ( ) <EOL> connected_component . append ( successor ) <EOL> if successor == node : break <EOL> component = tuple ( connected_component ) <EOL> result . append ( component ) <EOL> for node in graph : <EOL> if node not in lowlinks : <EOL> strongconnect ( node ) <EOL> return result <EOL> @ property <EOL> def dot ( self ) : <EOL> result = [ '<STR_LIT>' ] <EOL> for succ in self . _preds : <EOL> preds = self . _preds [ succ ] <EOL> for pred in preds : <EOL> result . append ( '<STR_LIT>' % ( pred , succ ) ) <EOL> for node in self . _nodes : <EOL> result . append ( '<STR_LIT>' % node ) <EOL> result . append ( '<STR_LIT:}>' ) <EOL> return '<STR_LIT:\n>' . join ( result ) <EOL> ARCHIVE_EXTENSIONS = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def unarchive ( archive_filename , dest_dir , format = None , check = True ) : <EOL> def check_path ( path ) : <EOL> if not isinstance ( path , text_type ) : <EOL> path = path . decode ( '<STR_LIT:utf-8>' ) <EOL> p = os . path . abspath ( os . path . join ( dest_dir , path ) ) <EOL> if not p . startswith ( dest_dir ) or p [ plen ] != os . sep : <EOL> raise ValueError ( '<STR_LIT>' % p ) <EOL> dest_dir = os . path . abspath ( dest_dir ) <EOL> plen = len ( dest_dir ) <EOL> archive = None <EOL> if format is None : <EOL> if archive_filename . endswith ( ( '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> format = '<STR_LIT>' <EOL> elif archive_filename . endswith ( ( '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> format = '<STR_LIT>' <EOL> mode = '<STR_LIT>' <EOL> elif archive_filename . endswith ( ( '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> format = '<STR_LIT>' <EOL> mode = '<STR_LIT>' <EOL> elif archive_filename . endswith ( '<STR_LIT>' ) : <EOL> format = '<STR_LIT>' <EOL> mode = '<STR_LIT:r>' <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' % archive_filename ) <EOL> try : <EOL> if format == '<STR_LIT>' : <EOL> archive = ZipFile ( archive_filename , '<STR_LIT:r>' ) <EOL> if check : <EOL> names = archive . namelist ( ) <EOL> for name in names : <EOL> check_path ( name ) <EOL> else : <EOL> archive = tarfile . open ( archive_filename , mode ) <EOL> if check : <EOL> names = archive . getnames ( ) <EOL> for name in names : <EOL> check_path ( name ) <EOL> if format != '<STR_LIT>' and sys . version_info [ <NUM_LIT:0> ] < <NUM_LIT:3> : <EOL> for tarinfo in archive . getmembers ( ) : <EOL> if not isinstance ( tarinfo . name , text_type ) : <EOL> tarinfo . name = tarinfo . name . decode ( '<STR_LIT:utf-8>' ) <EOL> archive . extractall ( dest_dir ) <EOL> finally : <EOL> if archive : <EOL> archive . close ( ) <EOL> def zip_dir ( directory ) : <EOL> """<STR_LIT>""" <EOL> result = io . BytesIO ( ) <EOL> dlen = len ( directory ) <EOL> with ZipFile ( result , "<STR_LIT:w>" ) as zf : <EOL> for root , dirs , files in os . walk ( directory ) : <EOL> for name in files : <EOL> full = os . path . join ( root , name ) <EOL> rel = root [ dlen : ] <EOL> dest = os . path . join ( rel , name ) <EOL> zf . write ( full , dest ) <EOL> return result <EOL> UNITS = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:M>' , '<STR_LIT>' , '<STR_LIT:T>' , '<STR_LIT:P>' ) <EOL> class Progress ( object ) : <EOL> unknown = '<STR_LIT>' <EOL> def __init__ ( self , minval = <NUM_LIT:0> , maxval = <NUM_LIT:100> ) : <EOL> assert maxval is None or maxval >= minval <EOL> self . min = self . cur = minval <EOL> self . max = maxval <EOL> self . started = None <EOL> self . elapsed = <NUM_LIT:0> <EOL> self . done = False <EOL> def update ( self , curval ) : <EOL> assert self . min <= curval <EOL> assert self . max is None or curval <= self . max <EOL> self . cur = curval <EOL> now = time . time ( ) <EOL> if self . started is None : <EOL> self . started = now <EOL> else : <EOL> self . elapsed = now - self . started <EOL> def increment ( self , incr ) : <EOL> assert incr >= <NUM_LIT:0> <EOL> self . update ( self . cur + incr ) <EOL> def start ( self ) : <EOL> self . update ( self . min ) <EOL> return self <EOL> def stop ( self ) : <EOL> if self . max is not None : <EOL> self . update ( self . max ) <EOL> self . done = True <EOL> @ property <EOL> def maximum ( self ) : <EOL> return self . unknown if self . max is None else self . max <EOL> @ property <EOL> def percentage ( self ) : <EOL> if self . done : <EOL> result = '<STR_LIT>' <EOL> elif self . max is None : <EOL> result = '<STR_LIT>' <EOL> else : <EOL> v = <NUM_LIT> * ( self . cur - self . min ) / ( self . max - self . min ) <EOL> result = '<STR_LIT>' % v <EOL> return result <EOL> def format_duration ( self , duration ) : <EOL> if ( duration <= <NUM_LIT:0> ) and self . max is None or self . cur == self . min : <EOL> result = '<STR_LIT>' <EOL> else : <EOL> result = time . strftime ( '<STR_LIT>' , time . gmtime ( duration ) ) <EOL> return result <EOL> @ property <EOL> def ETA ( self ) : <EOL> if self . done : <EOL> prefix = '<STR_LIT>' <EOL> t = self . elapsed <EOL> else : <EOL> prefix = '<STR_LIT>' <EOL> if self . max is None : <EOL> t = - <NUM_LIT:1> <EOL> elif self . elapsed == <NUM_LIT:0> or ( self . cur == self . min ) : <EOL> t = <NUM_LIT:0> <EOL> else : <EOL> t = float ( self . max - self . min ) <EOL> t /= self . cur - self . min <EOL> t = ( t - <NUM_LIT:1> ) * self . elapsed <EOL> return '<STR_LIT>' % ( prefix , self . format_duration ( t ) ) <EOL> @ property <EOL> def speed ( self ) : <EOL> if self . elapsed == <NUM_LIT:0> : <EOL> result = <NUM_LIT:0.0> <EOL> else : <EOL> result = ( self . cur - self . min ) / self . elapsed <EOL> for unit in UNITS : <EOL> if result < <NUM_LIT:1000> : <EOL> break <EOL> result /= <NUM_LIT> <EOL> return '<STR_LIT>' % ( result , unit ) <EOL> RICH_GLOB = re . compile ( r'<STR_LIT>' ) <EOL> _CHECK_RECURSIVE_GLOB = re . compile ( r'<STR_LIT>' ) <EOL> _CHECK_MISMATCH_SET = re . compile ( r'<STR_LIT>' ) <EOL> def iglob ( path_glob ) : <EOL> """<STR_LIT>""" <EOL> if _CHECK_RECURSIVE_GLOB . search ( path_glob ) : <EOL> msg = """<STR_LIT>""" <EOL> raise ValueError ( msg % path_glob ) <EOL> if _CHECK_MISMATCH_SET . search ( path_glob ) : <EOL> msg = """<STR_LIT>""" <EOL> raise ValueError ( msg % path_glob ) <EOL> return _iglob ( path_glob ) <EOL> def _iglob ( path_glob ) : <EOL> rich_path_glob = RICH_GLOB . split ( path_glob , <NUM_LIT:1> ) <EOL> if len ( rich_path_glob ) > <NUM_LIT:1> : <EOL> assert len ( rich_path_glob ) == <NUM_LIT:3> , rich_path_glob <EOL> prefix , set , suffix = rich_path_glob <EOL> for item in set . split ( '<STR_LIT:U+002C>' ) : <EOL> for path in _iglob ( '<STR_LIT>' . join ( ( prefix , item , suffix ) ) ) : <EOL> yield path <EOL> else : <EOL> if '<STR_LIT>' not in path_glob : <EOL> for item in std_iglob ( path_glob ) : <EOL> yield item <EOL> else : <EOL> prefix , radical = path_glob . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> if prefix == '<STR_LIT>' : <EOL> prefix = '<STR_LIT:.>' <EOL> if radical == '<STR_LIT>' : <EOL> radical = '<STR_LIT:*>' <EOL> else : <EOL> radical = radical . lstrip ( '<STR_LIT:/>' ) <EOL> radical = radical . lstrip ( '<STR_LIT:\\>' ) <EOL> for path , dir , files in os . walk ( prefix ) : <EOL> path = os . path . normpath ( path ) <EOL> for fn in _iglob ( os . path . join ( path , radical ) ) : <EOL> yield fn <EOL> class HTTPSConnection ( httplib . HTTPSConnection ) : <EOL> ca_certs = None <EOL> check_domain = True <EOL> def connect ( self ) : <EOL> sock = socket . create_connection ( ( self . host , self . port ) , self . timeout ) <EOL> if getattr ( self , '<STR_LIT>' , False ) : <EOL> self . sock = sock <EOL> self . _tunnel ( ) <EOL> if not hasattr ( ssl , '<STR_LIT>' ) : <EOL> if self . ca_certs : <EOL> cert_reqs = ssl . CERT_REQUIRED <EOL> else : <EOL> cert_reqs = ssl . CERT_NONE <EOL> self . sock = ssl . wrap_socket ( sock , self . key_file , self . cert_file , <EOL> cert_reqs = cert_reqs , <EOL> ssl_version = ssl . PROTOCOL_SSLv23 , <EOL> ca_certs = self . ca_certs ) <EOL> else : <EOL> context = ssl . SSLContext ( ssl . PROTOCOL_SSLv23 ) <EOL> context . options |= ssl . OP_NO_SSLv2 <EOL> if self . cert_file : <EOL> context . load_cert_chain ( self . cert_file , self . key_file ) <EOL> kwargs = { } <EOL> if self . ca_certs : <EOL> context . verify_mode = ssl . CERT_REQUIRED <EOL> context . load_verify_locations ( cafile = self . ca_certs ) <EOL> if getattr ( ssl , '<STR_LIT>' , False ) : <EOL> kwargs [ '<STR_LIT>' ] = self . host <EOL> self . sock = context . wrap_socket ( sock , ** kwargs ) <EOL> if self . ca_certs and self . check_domain : <EOL> try : <EOL> match_hostname ( self . sock . getpeercert ( ) , self . host ) <EOL> logger . debug ( '<STR_LIT>' , self . host ) <EOL> except CertificateError : <EOL> self . sock . shutdown ( socket . SHUT_RDWR ) <EOL> self . sock . close ( ) <EOL> raise <EOL> class HTTPSHandler ( BaseHTTPSHandler ) : <EOL> def __init__ ( self , ca_certs , check_domain = True ) : <EOL> BaseHTTPSHandler . __init__ ( self ) <EOL> self . ca_certs = ca_certs <EOL> self . check_domain = check_domain <EOL> def _conn_maker ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> result = HTTPSConnection ( * args , ** kwargs ) <EOL> if self . ca_certs : <EOL> result . ca_certs = self . ca_certs <EOL> result . check_domain = self . check_domain <EOL> return result <EOL> def https_open ( self , req ) : <EOL> try : <EOL> return self . do_open ( self . _conn_maker , req ) <EOL> except URLError as e : <EOL> if '<STR_LIT>' in str ( e . reason ) : <EOL> raise CertificateError ( '<STR_LIT>' <EOL> '<STR_LIT>' % req . host ) <EOL> else : <EOL> raise <EOL> class HTTPSOnlyHandler ( HTTPSHandler , HTTPHandler ) : <EOL> def http_open ( self , req ) : <EOL> raise URLError ( '<STR_LIT>' <EOL> '<STR_LIT>' % req ) <EOL> _ver_info = sys . version_info [ : <NUM_LIT:2> ] <EOL> if _ver_info == ( <NUM_LIT:2> , <NUM_LIT:6> ) : <EOL> class HTTP ( httplib . HTTP ) : <EOL> def __init__ ( self , host = '<STR_LIT>' , port = None , ** kwargs ) : <EOL> if port == <NUM_LIT:0> : <EOL> port = None <EOL> self . _setup ( self . _connection_class ( host , port , ** kwargs ) ) <EOL> class HTTPS ( httplib . HTTPS ) : <EOL> def __init__ ( self , host = '<STR_LIT>' , port = None , ** kwargs ) : <EOL> if port == <NUM_LIT:0> : <EOL> port = None <EOL> self . _setup ( self . _connection_class ( host , port , ** kwargs ) ) <EOL> class Transport ( xmlrpclib . Transport ) : <EOL> def __init__ ( self , timeout , use_datetime = <NUM_LIT:0> ) : <EOL> self . timeout = timeout <EOL> xmlrpclib . Transport . __init__ ( self , use_datetime ) <EOL> def make_connection ( self , host ) : <EOL> h , eh , x509 = self . get_host_info ( host ) <EOL> if _ver_info == ( <NUM_LIT:2> , <NUM_LIT:6> ) : <EOL> result = HTTP ( h , timeout = self . timeout ) <EOL> else : <EOL> if not self . _connection or host != self . _connection [ <NUM_LIT:0> ] : <EOL> self . _extra_headers = eh <EOL> self . _connection = host , httplib . HTTPConnection ( h ) <EOL> result = self . _connection [ <NUM_LIT:1> ] <EOL> return result <EOL> class SafeTransport ( xmlrpclib . SafeTransport ) : <EOL> def __init__ ( self , timeout , use_datetime = <NUM_LIT:0> ) : <EOL> self . timeout = timeout <EOL> xmlrpclib . SafeTransport . __init__ ( self , use_datetime ) <EOL> def make_connection ( self , host ) : <EOL> h , eh , kwargs = self . get_host_info ( host ) <EOL> if not kwargs : <EOL> kwargs = { } <EOL> kwargs [ '<STR_LIT>' ] = self . timeout <EOL> if _ver_info == ( <NUM_LIT:2> , <NUM_LIT:6> ) : <EOL> result = HTTPS ( host , None , ** kwargs ) <EOL> else : <EOL> if not self . _connection or host != self . _connection [ <NUM_LIT:0> ] : <EOL> self . _extra_headers = eh <EOL> self . _connection = host , httplib . HTTPSConnection ( h , None , <EOL> ** kwargs ) <EOL> result = self . _connection [ <NUM_LIT:1> ] <EOL> return result <EOL> class ServerProxy ( xmlrpclib . ServerProxy ) : <EOL> def __init__ ( self , uri , ** kwargs ) : <EOL> self . timeout = timeout = kwargs . pop ( '<STR_LIT>' , None ) <EOL> if timeout is not None : <EOL> scheme , _ = splittype ( uri ) <EOL> use_datetime = kwargs . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> if scheme == '<STR_LIT>' : <EOL> tcls = SafeTransport <EOL> else : <EOL> tcls = Transport <EOL> kwargs [ '<STR_LIT>' ] = t = tcls ( timeout , use_datetime = use_datetime ) <EOL> self . transport = t <EOL> xmlrpclib . ServerProxy . __init__ ( self , uri , ** kwargs ) <EOL> def _csv_open ( fn , mode , ** kwargs ) : <EOL> if sys . version_info [ <NUM_LIT:0> ] < <NUM_LIT:3> : <EOL> mode += '<STR_LIT:b>' <EOL> else : <EOL> kwargs [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> return open ( fn , mode , ** kwargs ) <EOL> class CSVBase ( object ) : <EOL> defaults = { <EOL> '<STR_LIT>' : str ( '<STR_LIT:U+002C>' ) , <EOL> '<STR_LIT>' : str ( '<STR_LIT:">' ) , <EOL> '<STR_LIT>' : str ( '<STR_LIT:\n>' ) <EOL> } <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , * exc_info ) : <EOL> self . stream . close ( ) <EOL> class CSVReader ( CSVBase ) : <EOL> def __init__ ( self , ** kwargs ) : <EOL> if '<STR_LIT>' in kwargs : <EOL> stream = kwargs [ '<STR_LIT>' ] <EOL> if sys . version_info [ <NUM_LIT:0> ] >= <NUM_LIT:3> : <EOL> stream = codecs . getreader ( '<STR_LIT:utf-8>' ) ( stream ) <EOL> self . stream = stream <EOL> else : <EOL> self . stream = _csv_open ( kwargs [ '<STR_LIT:path>' ] , '<STR_LIT:r>' ) <EOL> self . reader = csv . reader ( self . stream , ** self . defaults ) <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> result = next ( self . reader ) <EOL> if sys . version_info [ <NUM_LIT:0> ] < <NUM_LIT:3> : <EOL> for i , item in enumerate ( result ) : <EOL> if not isinstance ( item , text_type ) : <EOL> result [ i ] = item . decode ( '<STR_LIT:utf-8>' ) <EOL> return result <EOL> __next__ = next <EOL> class CSVWriter ( CSVBase ) : <EOL> def __init__ ( self , fn , ** kwargs ) : <EOL> self . stream = _csv_open ( fn , '<STR_LIT:w>' ) <EOL> self . writer = csv . writer ( self . stream , ** self . defaults ) <EOL> def writerow ( self , row ) : <EOL> if sys . version_info [ <NUM_LIT:0> ] < <NUM_LIT:3> : <EOL> r = [ ] <EOL> for item in row : <EOL> if isinstance ( item , text_type ) : <EOL> item = item . encode ( '<STR_LIT:utf-8>' ) <EOL> r . append ( item ) <EOL> row = r <EOL> self . writer . writerow ( row ) <EOL> class Configurator ( BaseConfigurator ) : <EOL> value_converters = dict ( BaseConfigurator . value_converters ) <EOL> value_converters [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> def __init__ ( self , config , base = None ) : <EOL> super ( Configurator , self ) . __init__ ( config ) <EOL> self . base = base or os . getcwd ( ) <EOL> def configure_custom ( self , config ) : <EOL> def convert ( o ) : <EOL> if isinstance ( o , ( list , tuple ) ) : <EOL> result = type ( o ) ( [ convert ( i ) for i in o ] ) <EOL> elif isinstance ( o , dict ) : <EOL> if '<STR_LIT>' in o : <EOL> result = self . configure_custom ( o ) <EOL> else : <EOL> result = { } <EOL> for k in o : <EOL> result [ k ] = convert ( o [ k ] ) <EOL> else : <EOL> result = self . convert ( o ) <EOL> return result <EOL> c = config . pop ( '<STR_LIT>' ) <EOL> if not callable ( c ) : <EOL> c = self . resolve ( c ) <EOL> props = config . pop ( '<STR_LIT:.>' , None ) <EOL> args = config . pop ( '<STR_LIT>' , ( ) ) <EOL> if args : <EOL> args = tuple ( [ convert ( o ) for o in args ] ) <EOL> items = [ ( k , convert ( config [ k ] ) ) for k in config if valid_ident ( k ) ] <EOL> kwargs = dict ( items ) <EOL> result = c ( * args , ** kwargs ) <EOL> if props : <EOL> for n , v in props . items ( ) : <EOL> setattr ( result , n , convert ( v ) ) <EOL> return result <EOL> def __getitem__ ( self , key ) : <EOL> result = self . config [ key ] <EOL> if isinstance ( result , dict ) and '<STR_LIT>' in result : <EOL> self . config [ key ] = result = self . configure_custom ( result ) <EOL> return result <EOL> def inc_convert ( self , value ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isabs ( value ) : <EOL> value = os . path . join ( self . base , value ) <EOL> with codecs . open ( value , '<STR_LIT:r>' , encoding = '<STR_LIT:utf-8>' ) as f : <EOL> result = json . load ( f ) <EOL> return result <EOL> class SubprocessMixin ( object ) : <EOL> def __init__ ( self , verbose = False , progress = None ) : <EOL> self . verbose = verbose <EOL> self . progress = progress <EOL> def reader ( self , stream , context ) : <EOL> """<STR_LIT>""" <EOL> progress = self . progress <EOL> verbose = self . verbose <EOL> while True : <EOL> s = stream . readline ( ) <EOL> if not s : <EOL> break <EOL> if progress is not None : <EOL> progress ( s , context ) <EOL> else : <EOL> if not verbose : <EOL> sys . stderr . write ( '<STR_LIT:.>' ) <EOL> else : <EOL> sys . stderr . write ( s . decode ( '<STR_LIT:utf-8>' ) ) <EOL> sys . stderr . flush ( ) <EOL> stream . close ( ) <EOL> def run_command ( self , cmd , ** kwargs ) : <EOL> p = subprocess . Popen ( cmd , stdout = subprocess . PIPE , <EOL> stderr = subprocess . PIPE , ** kwargs ) <EOL> t1 = threading . Thread ( target = self . reader , args = ( p . stdout , '<STR_LIT>' ) ) <EOL> t1 . start ( ) <EOL> t2 = threading . Thread ( target = self . reader , args = ( p . stderr , '<STR_LIT>' ) ) <EOL> t2 . start ( ) <EOL> p . wait ( ) <EOL> t1 . join ( ) <EOL> t2 . join ( ) <EOL> if self . progress is not None : <EOL> self . progress ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> elif self . verbose : <EOL> sys . stderr . write ( '<STR_LIT>' ) <EOL> return p </s>
