<s> '''<STR_LIT>''' <EOL> from __future__ import with_statement <EOL> import sys , re , os , subprocess , shutil , hashlib , binascii , random , time , logging , optparse , functools , ConfigParser , calendar , cStringIO , errno , httplib , rfc822 <EOL> from datetime import datetime <EOL> from contextlib import contextmanager <EOL> import boto <EOL> from boto . s3 . connection import S3Connection <EOL> ZINC_VERSION = "<STR_LIT>" <EOL> REPO_VERSION = "<STR_LIT>" <EOL> REPO_VERSIONS_SUPPORTED = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> BOTO_VERSION = boto . Version <EOL> TMP_DIR = "<STR_LIT>" <EOL> REV_BITS = <NUM_LIT:64> <EOL> DATETIME_FORMAT = '<STR_LIT>' <EOL> UNKNOWN_USER = "<STR_LIT>" <EOL> EMPTY_MESSAGE = "<STR_LIT>" <EOL> EMPTY_FILEPATH = "<STR_LIT>" <EOL> BACKUP_SUFFIX = "<STR_LIT>" <EOL> TMP_DIR_PREFIX = "<STR_LIT>" <EOL> SCOPE_SEP = "<STR_LIT>" <EOL> S3_SINGLEPART_MAX_SIZE = <NUM_LIT> <EOL> S3_MULTIPART_CHUNK_SIZE = <NUM_LIT:50> << <NUM_LIT:20> <EOL> S3_ACCESS_KEY_NAME = "<STR_LIT>" <EOL> S3_SECRET_KEY_NAME = "<STR_LIT>" <EOL> ZINC_S3_MD5_NAME = "<STR_LIT>" <EOL> INVALID_MD5 = "<STR_LIT>" <EOL> INVALID_SIZE = - <NUM_LIT:1> <EOL> INVALID_MTIME = - <NUM_LIT:1.0> <EOL> SCHEME_DEFAULT = "<STR_LIT>" <EOL> LZO_LEVEL = <NUM_LIT:7> <EOL> DIFF_COMMAND = '<STR_LIT>' <EOL> def log_setup ( verbosity = <NUM_LIT:1> ) : <EOL> '''<STR_LIT>''' <EOL> global log <EOL> log = logging . Logger ( "<STR_LIT>" ) <EOL> log . verbosity = verbosity <EOL> log . stream = sys . stderr <EOL> log_handler = logging . StreamHandler ( log . stream ) <EOL> log . addHandler ( log_handler ) <EOL> if verbosity == <NUM_LIT:3> : <EOL> log_handler . setFormatter ( logging . Formatter ( '<STR_LIT>' , DATETIME_FORMAT ) ) <EOL> log . setLevel ( logging . DEBUG ) <EOL> elif verbosity == <NUM_LIT:2> : <EOL> log_handler . setFormatter ( logging . Formatter ( '<STR_LIT>' , DATETIME_FORMAT ) ) <EOL> log . setLevel ( logging . DEBUG ) <EOL> elif verbosity == <NUM_LIT:0> : <EOL> log_handler . setFormatter ( logging . Formatter ( '<STR_LIT>' , DATETIME_FORMAT ) ) <EOL> log . setLevel ( logging . WARN ) <EOL> else : <EOL> log_handler . setFormatter ( logging . Formatter ( '<STR_LIT>' , DATETIME_FORMAT ) ) <EOL> log . setLevel ( logging . INFO ) <EOL> log_setup ( ) <EOL> def general_setup ( ) : <EOL> import warnings <EOL> warnings . simplefilter ( "<STR_LIT:ignore>" , DeprecationWarning ) <EOL> log_setup ( ) <EOL> class Failure ( RuntimeError ) : <EOL> def __init__ ( self , message , suppressable = False ) : <EOL> self . suppressable = suppressable <EOL> RuntimeError . __init__ ( self , message ) <EOL> class InvalidArgument ( Failure ) : <EOL> pass <EOL> class InvalidOperation ( Failure ) : <EOL> pass <EOL> class FileExists ( InvalidOperation ) : <EOL> pass <EOL> class ShellError ( InvalidOperation ) : <EOL> pass <EOL> class BadState ( Failure ) : <EOL> pass <EOL> class MissingFile ( Failure ) : <EOL> pass <EOL> class _Enum ( object ) : <EOL> key_dict = { } <EOL> value_dict = { } <EOL> @ classmethod <EOL> def has_key ( cls , key ) : <EOL> return key in cls . key_dict <EOL> @ classmethod <EOL> def has_value ( cls , value ) : <EOL> return value in cls . value_dict <EOL> @ classmethod <EOL> def value_for ( cls , key ) : <EOL> if key in cls . value_dict : <EOL> return cls . key_dict [ key ] <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( cls . __name__ , key ) ) <EOL> @ classmethod <EOL> def key_for ( cls , value ) : <EOL> if value in cls . value_dict : <EOL> return cls . value_dict [ value ] <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( cls . __name__ , value ) ) <EOL> @ classmethod <EOL> def check_value ( cls , value ) : <EOL> cls . key_for ( value ) <EOL> return value <EOL> @ classmethod <EOL> def values ( cls ) : <EOL> return cls . value_dict . keys ( ) <EOL> def enum ( name , ** enum_dict ) : <EOL> '''<STR_LIT>''' <EOL> new_type = type ( name , ( _Enum , ) , enum_dict ) <EOL> new_type . key_dict = enum_dict <EOL> new_type . value_dict = dict ( ( v , k ) for ( k , v ) in enum_dict . iteritems ( ) ) <EOL> return new_type <EOL> def fail ( message = None , prefix = "<STR_LIT:error>" , exc_info = None , status = <NUM_LIT:1> , suppress = False ) : <EOL> '''<STR_LIT>''' <EOL> if suppress : <EOL> if message : <EOL> log . info ( "<STR_LIT>" , message ) <EOL> sys . exit ( <NUM_LIT:0> ) <EOL> else : <EOL> if message : <EOL> log . error ( "<STR_LIT>" , prefix , message ) <EOL> if exc_info and log . verbosity >= <NUM_LIT:1> : <EOL> log . info ( "<STR_LIT>" , exc_info = exc_info ) <EOL> sys . exit ( status ) <EOL> def log_calls_with ( severity ) : <EOL> '''<STR_LIT>''' <EOL> def decorator ( fn ) : <EOL> @ functools . wraps ( fn ) <EOL> def wrap ( * params , ** kwargs ) : <EOL> call_str = "<STR_LIT>" % ( fn . __name__ , "<STR_LIT:U+002CU+0020>" . join ( [ repr ( p ) for p in params ] + [ "<STR_LIT>" % ( k , repr ( v ) ) for ( k , v ) in kwargs . items ( ) ] ) ) <EOL> log . log ( severity , "<STR_LIT>" , call_str ) <EOL> ret = fn ( * params , ** kwargs ) <EOL> log . log ( severity , "<STR_LIT>" , call_str , repr ( ret ) ) <EOL> return ret <EOL> return wrap <EOL> return decorator <EOL> log_calls_info = log_calls_with ( logging . INFO ) <EOL> log_calls = log_calls_with ( logging . DEBUG ) <EOL> def split_uri ( uri_str ) : <EOL> '''<STR_LIT>''' <EOL> m = re . match ( "<STR_LIT>" , uri_str ) <EOL> if not m : <EOL> raise InvalidArgument ( "<STR_LIT>" % uri_str ) <EOL> return ( m . group ( <NUM_LIT:1> ) , m . group ( <NUM_LIT:2> ) ) <EOL> _WHITESPACE_RE = re . compile ( "<STR_LIT>" ) <EOL> def split_whitespace ( str ) : <EOL> '''<STR_LIT>''' <EOL> return re . split ( _WHITESPACE_RE , str ) <EOL> def parse_rev_range ( rev_str ) : <EOL> '''<STR_LIT>''' <EOL> l = [ s . strip ( ) for s in rev_str . split ( "<STR_LIT::>" ) ] <EOL> if len ( l ) == <NUM_LIT:2> : <EOL> return ( l [ <NUM_LIT:0> ] if l [ <NUM_LIT:0> ] else None , l [ <NUM_LIT:1> ] if l [ <NUM_LIT:1> ] else None ) <EOL> elif len ( l ) == <NUM_LIT:1> : <EOL> return ( l [ <NUM_LIT:0> ] , None ) <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % rev_str ) <EOL> def determine_rev_range ( options , config ) : <EOL> '''<STR_LIT>''' <EOL> rev = "<STR_LIT>" <EOL> if options . work_dir : <EOL> rev = WorkingDir ( options . work_dir , config ) . get_checkout_rev ( options . scope ) or rev <EOL> return parse_rev_range ( options . rev ) if options . rev else ( rev , None ) <EOL> def join_path ( * elements ) : <EOL> '''<STR_LIT>''' <EOL> if elements and elements [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> return join_path ( * elements [ <NUM_LIT:1> : ] ) <EOL> else : <EOL> return "<STR_LIT:/>" . join ( elements ) <EOL> _RANDOM = random . Random ( ) <EOL> _RANDOM . seed ( ) <EOL> def new_uid ( bits = <NUM_LIT:64> ) : <EOL> '''<STR_LIT>''' <EOL> return "<STR_LIT>" . join ( _RANDOM . sample ( "<STR_LIT>" , int ( bits / <NUM_LIT> ) + <NUM_LIT:1> ) ) <EOL> def group_pairs ( pairs ) : <EOL> out = { } <EOL> for ( x , y ) in pairs : <EOL> if x in out : <EOL> out [ x ] . append ( y ) <EOL> else : <EOL> out [ x ] = [ y ] <EOL> return out <EOL> def setattrs ( self , others ) : <EOL> '''<STR_LIT>''' <EOL> for ( k , v ) in others . iteritems ( ) : <EOL> setattr ( self , k , v ) <EOL> def make_all_dirs ( path ) : <EOL> '''<STR_LIT>''' <EOL> if not os . path . isdir ( path ) : <EOL> os . makedirs ( path ) <EOL> def make_parent_dirs ( path ) : <EOL> '''<STR_LIT>''' <EOL> dir = os . path . dirname ( path ) <EOL> if dir and dir != "<STR_LIT:/>" : <EOL> make_all_dirs ( dir ) <EOL> def set_file_mtime ( path , mtime , atime = None ) : <EOL> '''<STR_LIT>''' <EOL> if not atime : <EOL> atime = mtime <EOL> f = file ( path , '<STR_LIT:a>' ) <EOL> try : <EOL> os . utime ( path , ( atime , mtime ) ) <EOL> finally : <EOL> f . close ( ) <EOL> def move_to_backup ( path , backup_suffix = BACKUP_SUFFIX ) : <EOL> if backup_suffix and os . path . exists ( path ) : <EOL> shutil . move ( path , path + backup_suffix ) <EOL> def parent_dirs ( path ) : <EOL> '''<STR_LIT>''' <EOL> root = "<STR_LIT:/>" if path . startswith ( "<STR_LIT:/>" ) else "<STR_LIT>" <EOL> path = path . strip ( "<STR_LIT:/>" ) <EOL> elements = path . split ( "<STR_LIT:/>" ) <EOL> right_elements = [ ] <EOL> while len ( elements ) > <NUM_LIT:0> : <EOL> parent = "<STR_LIT:/>" . join ( elements ) <EOL> rel_path = "<STR_LIT:/>" . join ( right_elements ) <EOL> yield ( root + parent , rel_path ) <EOL> right_elements . insert ( <NUM_LIT:0> , elements . pop ( ) ) <EOL> yield ( root , path ) <EOL> def new_temp_filename ( prefix = None , based_on = None , with_nonce = True ) : <EOL> '''<STR_LIT>''' <EOL> if based_on : <EOL> if not prefix : <EOL> prefix = "<STR_LIT>" <EOL> temp_path = "<STR_LIT>" % ( os . path . dirname ( based_on ) or "<STR_LIT:.>" , prefix , os . path . basename ( based_on ) ) <EOL> else : <EOL> temp_path = "<STR_LIT>" % TMP_DIR <EOL> if prefix : <EOL> temp_path = "<STR_LIT>" % ( temp_path , prefix ) <EOL> if with_nonce : <EOL> temp_path = "<STR_LIT>" % ( temp_path , new_uid ( ) ) <EOL> return temp_path <EOL> @ contextmanager <EOL> def atomic_output_file ( dest_path , make_parents = False , backup_suffix = None , suffix = "<STR_LIT>" ) : <EOL> '''<STR_LIT>''' <EOL> tmp_path = ( "<STR_LIT:%s>" + suffix ) % ( dest_path , new_uid ( ) ) <EOL> if make_parents : <EOL> make_parent_dirs ( tmp_path ) <EOL> yield tmp_path <EOL> if not os . path . isfile ( tmp_path ) : <EOL> raise BadState ( "<STR_LIT>" % ( dest_path , tmp_path ) ) <EOL> move_to_backup ( dest_path , backup_suffix = backup_suffix ) <EOL> shutil . move ( tmp_path , dest_path ) <EOL> @ contextmanager <EOL> def temp_output_file ( prefix = None , based_on = None , with_nonce = True ) : <EOL> '''<STR_LIT>''' <EOL> tmp_path = new_temp_filename ( prefix = prefix , based_on = based_on , with_nonce = with_nonce ) <EOL> yield tmp_path <EOL> try : <EOL> os . remove ( tmp_path ) <EOL> except OSError , e : <EOL> pass <EOL> @ log_calls <EOL> def copyfile_atomic ( source_path , dest_path , make_parents = False , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> with atomic_output_file ( dest_path , make_parents = make_parents , backup_suffix = backup_suffix ) as tmp_path : <EOL> shutil . copyfile ( source_path , tmp_path ) <EOL> set_file_mtime ( tmp_path , os . path . getmtime ( source_path ) ) <EOL> def movefile ( source_path , dest_path , make_parents = False , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> if make_parents : <EOL> make_parent_dirs ( dest_path ) <EOL> move_to_backup ( dest_path , backup_suffix = backup_suffix ) <EOL> shutil . move ( source_path , dest_path ) <EOL> def read_string_from_file ( path ) : <EOL> '''<STR_LIT>''' <EOL> with open ( path , "<STR_LIT:rb>" ) as f : <EOL> value = f . read ( ) <EOL> return value <EOL> def write_string_to_file ( path , string , make_parents = False , backup_suffix = BACKUP_SUFFIX ) : <EOL> '''<STR_LIT>''' <EOL> with atomic_output_file ( path , make_parents = make_parents , backup_suffix = backup_suffix ) as tmp_path : <EOL> with open ( tmp_path , "<STR_LIT:wb>" ) as f : <EOL> f . write ( string ) <EOL> def file_md5 ( path , blocksize = <NUM_LIT:2> ** <NUM_LIT> ) : <EOL> '''<STR_LIT>''' <EOL> md5 = hashlib . md5 ( ) <EOL> size = os . path . getsize ( path ) <EOL> if size > <NUM_LIT:2> ** <NUM_LIT> : <EOL> log . debug ( "<STR_LIT>" , path , size ) <EOL> with open ( path , "<STR_LIT:rb>" ) as f : <EOL> while True : <EOL> data = f . read ( blocksize ) <EOL> if not data : <EOL> break <EOL> md5 . update ( data ) <EOL> value = binascii . hexlify ( md5 . digest ( ) ) <EOL> return value <EOL> def strip_prefix ( prefix , value ) : <EOL> '''<STR_LIT>''' <EOL> return value [ len ( prefix ) : ] if value . startswith ( prefix ) else None <EOL> def list_files_recursive ( path ) : <EOL> '''<STR_LIT>''' <EOL> for root , dirnames , filenames in os . walk ( path ) : <EOL> for filename in filenames : <EOL> val = strip_prefix ( path + "<STR_LIT:/>" , join_path ( root , filename ) ) <EOL> assert val <EOL> yield val <EOL> def list_dirs_recursive ( path ) : <EOL> '''<STR_LIT>''' <EOL> for root , dirnames , filenames in os . walk ( path , topdown = False ) : <EOL> for dirname in dirnames : <EOL> val = strip_prefix ( path + "<STR_LIT:/>" , join_path ( root , dirname ) ) <EOL> assert val <EOL> yield val <EOL> @ contextmanager <EOL> def temp_output_dir ( output_dir , temp_dir = None , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> if not temp_dir : <EOL> temp_dir = join_path ( output_dir , "<STR_LIT>" % ( TMP_DIR_PREFIX , new_uid ( ) ) ) <EOL> make_all_dirs ( temp_dir ) <EOL> yield temp_dir <EOL> for path in list_files_recursive ( temp_dir ) : <EOL> src = join_path ( temp_dir , path ) <EOL> dest = join_path ( output_dir , path ) <EOL> log . debug ( "<STR_LIT>" , src , dest ) <EOL> movefile ( src , dest , make_parents = True , backup_suffix = backup_suffix ) <EOL> finally : <EOL> try : <EOL> for dir in list_dirs_recursive ( temp_dir ) : <EOL> os . rmdir ( join_path ( temp_dir , dir ) ) <EOL> os . rmdir ( temp_dir ) <EOL> except Exception , e : <EOL> log . warn ( "<STR_LIT>" , temp_dir , e ) <EOL> def shell_command ( command , wait = True ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> log . debug ( "<STR_LIT>" , "<STR_LIT:U+0020>" . join ( command ) ) <EOL> p = subprocess . Popen ( command , stdout = subprocess . PIPE , stderr = subprocess . STDOUT , shell = False ) <EOL> if wait : <EOL> p . wait ( ) <EOL> else : <EOL> return p <EOL> except subprocess . CalledProcessError , e : <EOL> raise ShellError ( "<STR_LIT>" % ( "<STR_LIT:U+0020>" . join ( command ) , e . returncode ) ) <EOL> except OSError , e : <EOL> raise ShellError ( "<STR_LIT>" % ( "<STR_LIT:U+0020>" . join ( command ) , e ) ) <EOL> def expand_wildcards ( wildcards , candidates , recursive = False ) : <EOL> '''<STR_LIT>''' <EOL> out = [ ] <EOL> match_char = '<STR_LIT:.>' if recursive else '<STR_LIT>' <EOL> for wildcard in wildcards : <EOL> if '<STR_LIT:*>' in wildcard or '<STR_LIT:?>' in wildcard : <EOL> pattern = r'<STR_LIT>' + re . escape ( wildcard ) . replace ( '<STR_LIT>' , match_char + '<STR_LIT:*>' ) . replace ( '<STR_LIT>' , match_char ) + r'<STR_LIT:$>' <EOL> out . extend ( [ candidate for candidate in candidates if re . search ( pattern , candidate ) is not None ] ) <EOL> else : <EOL> out . append ( wildcard ) <EOL> return out <EOL> def color_text ( text , color ) : <EOL> colors = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> return "<STR_LIT>" . join ( [ colors [ color ] , text , colors [ '<STR_LIT>' ] ] ) <EOL> def print_diff ( from_path , to_path , from_label = None , to_label = None ) : <EOL> if not from_label : <EOL> from_label = from_path <EOL> if not to_label : <EOL> to_label = to_path <EOL> command = [ piece % { "<STR_LIT>" : from_label , "<STR_LIT>" : from_path , "<STR_LIT>" : to_label , "<STR_LIT>" : to_path } for piece in DIFF_COMMAND . split ( ) ] <EOL> p = shell_command ( command , wait = False ) <EOL> use_colors = sys . stdout . isatty ( ) <EOL> for line in p . stdout : <EOL> if use_colors : <EOL> if line . startswith ( '<STR_LIT:+>' ) : <EOL> line = color_text ( line , '<STR_LIT>' ) <EOL> elif line . startswith ( '<STR_LIT:->' ) : <EOL> line = color_text ( line , '<STR_LIT>' ) <EOL> elif line . startswith ( '<STR_LIT>' ) : <EOL> line = color_text ( line , '<STR_LIT>' ) <EOL> sys . stdout . write ( line ) <EOL> sys . stdout . write ( "<STR_LIT:\n>" ) <EOL> def current_repo_time ( ) : <EOL> '''<STR_LIT>''' <EOL> return int ( time . time ( ) ) <EOL> def format_seconds ( seconds ) : <EOL> return datetime . utcfromtimestamp ( seconds ) . strftime ( DATETIME_FORMAT ) <EOL> DATETIME_FORMATS = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] <EOL> _DATETIME_SECONDS_RE = re . compile ( "<STR_LIT>" ) <EOL> def parse_timezone ( tz_str ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> tz_str = tz_str . lower ( ) <EOL> if tz_str [ <NUM_LIT:0> ] in "<STR_LIT>" and len ( tz_str ) == <NUM_LIT:5> and tz_str [ <NUM_LIT:1> : ] . isdigit ( ) : <EOL> sign = <NUM_LIT:1> if ( tz_str [ <NUM_LIT:0> ] == "<STR_LIT:+>" ) else - <NUM_LIT:1> <EOL> hours = int ( tz_str [ <NUM_LIT:1> : <NUM_LIT:3> ] ) <EOL> minutes = int ( tz_str [ <NUM_LIT:3> : <NUM_LIT:5> ] ) <EOL> return - sign * ( <NUM_LIT> * hours + minutes ) <EOL> elif tz_str == "<STR_LIT>" or tz_str == "<STR_LIT>" : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> return None <EOL> except ( ValueError , IndexError ) : <EOL> return None <EOL> def parse_datetime ( datetime_str , formats = DATETIME_FORMATS , tz_offset = None , assume_utc = False ) : <EOL> '''<STR_LIT>''' <EOL> if _DATETIME_SECONDS_RE . match ( datetime_str ) : <EOL> return int ( datetime_str ) <EOL> if tz_offset is None : <EOL> bits = split_whitespace ( datetime_str ) <EOL> if len ( bits ) > <NUM_LIT:1> : <EOL> tz_offset = parse_timezone ( bits [ - <NUM_LIT:1> ] ) <EOL> if tz_offset is not None : <EOL> datetime_str = "<STR_LIT:U+0020>" . join ( bits [ : - <NUM_LIT:1> ] ) <EOL> datetime_str = datetime_str . strip ( ) <EOL> for format in formats : <EOL> try : <EOL> tuple = time . strptime ( datetime_str , format ) <EOL> if tz_offset is not None : <EOL> seconds = calendar . timegm ( tuple ) <EOL> seconds += tz_offset * <NUM_LIT> <EOL> elif assume_utc : <EOL> seconds = calendar . timegm ( tuple ) <EOL> else : <EOL> seconds = time . mktime ( tuple ) <EOL> except ( ValueError , OverflowError ) : <EOL> continue <EOL> break <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % datetime_str ) <EOL> return seconds <EOL> _DATESPEC_RE1 = re . compile ( "<STR_LIT>" ) <EOL> _DATESPEC_RE1A = re . compile ( "<STR_LIT>" ) <EOL> _DATESPEC_RE2 = re . compile ( "<STR_LIT>" ) <EOL> _DATESPEC_RE2A = re . compile ( "<STR_LIT>" ) <EOL> def parse_datespec ( datespec_str , assume_utc = False ) : <EOL> '''<STR_LIT>''' <EOL> datespec_str = datespec_str . strip ( ) <EOL> m = None <EOL> if not m : <EOL> m = _DATESPEC_RE1 . match ( datespec_str ) or _DATESPEC_RE1A . match ( datespec_str ) <EOL> if m : <EOL> direction = m . group ( <NUM_LIT:1> ) <EOL> time = parse_datetime ( m . group ( <NUM_LIT:2> ) , assume_utc = assume_utc ) <EOL> if direction == "<STR_LIT:<>" : <EOL> filter = lambda t : t <= time <EOL> else : <EOL> filter = lambda t : t >= time <EOL> if not m : <EOL> m = _DATESPEC_RE2 . match ( datespec_str ) or _DATESPEC_RE2A . match ( datespec_str ) <EOL> if m : <EOL> time1 , time2 = parse_datetime ( m . group ( <NUM_LIT:1> ) ) , parse_datetime ( m . group ( <NUM_LIT:2> ) ) <EOL> filter = lambda t : time1 <= t and t <= time2 <EOL> if not m : <EOL> raise InvalidArgument ( "<STR_LIT>" % datespec_str ) <EOL> return filter <EOL> def shortvals_to_str ( shortvals ) : <EOL> '''<STR_LIT>''' <EOL> if type ( shortvals ) == dict : <EOL> shortvals = [ ( k , shortvals [ k ] ) for k in sorted ( shortvals ) ] <EOL> for ( k , v ) in shortvals : <EOL> if "<STR_LIT:;>" in str ( k ) or "<STR_LIT:=>" in str ( k ) or ( v is not None and ( "<STR_LIT:;>" in str ( v ) or "<STR_LIT:=>" in str ( v ) ) ) : <EOL> raise InvalidArgument ( "<STR_LIT>" % shortvals ) <EOL> return "<STR_LIT:;>" . join ( [ "<STR_LIT>" % ( str ( k ) , "<STR_LIT>" % v if isinstance ( v , float ) else str ( v ) ) for ( k , v ) in shortvals if v is not None ] ) <EOL> def shortvals_from_str ( shortvals_str ) : <EOL> '''<STR_LIT>''' <EOL> return dict ( pairstr . split ( "<STR_LIT:=>" ) for pairstr in shortvals_str . split ( "<STR_LIT:;>" ) ) <EOL> class Values ( object ) : <EOL> '''<STR_LIT>''' <EOL> RECORD_MAGIC = "<STR_LIT>" <EOL> VERSION = "<STR_LIT>" <EOL> def __init__ ( self , values = None , comment = None ) : <EOL> '''<STR_LIT>''' <EOL> if not values : <EOL> values = { } <EOL> self . values = [ ] <EOL> if type ( values ) == dict : <EOL> self . values = [ ( k , str ( values [ k ] ) ) for k in sorted ( values ) if values [ k ] is not None ] <EOL> elif type ( values ) == list : <EOL> self . values = [ ( k , str ( v ) ) for ( k , v ) in values if v is not None ] <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % type ( values ) ) <EOL> self . comment = comment <EOL> def from_str ( self , str_val ) : <EOL> lines = str_val . splitlines ( ) <EOL> cur_rec = None <EOL> cur_val = "<STR_LIT>" <EOL> for line in lines : <EOL> line = line . strip ( ) <EOL> if line . startswith ( "<STR_LIT:#>" ) : <EOL> continue <EOL> if line . startswith ( self . RECORD_MAGIC ) : <EOL> if cur_rec : <EOL> self . values . append ( ( cur_rec , cur_val [ : - <NUM_LIT:1> ] ) ) <EOL> cur_rec = line [ len ( self . RECORD_MAGIC ) : ] <EOL> cur_val = "<STR_LIT>" <EOL> continue <EOL> cur_val += line <EOL> cur_val += "<STR_LIT:\n>" <EOL> self . values . append ( ( cur_rec , cur_val [ : - <NUM_LIT:1> ] ) ) <EOL> def to_str ( self ) : <EOL> out = "<STR_LIT>" % self . VERSION <EOL> if self . comment : <EOL> for l in self . comment . splitlines ( ) : <EOL> out += "<STR_LIT>" % l <EOL> for ( key , value ) in self . values : <EOL> assert value is not None <EOL> out += self . RECORD_MAGIC + key + "<STR_LIT:\n>" <EOL> out += value <EOL> out += "<STR_LIT:\n>" <EOL> return out <EOL> def clear ( self ) : <EOL> self . values = { } <EOL> def values_to_str ( values , comment = None ) : <EOL> '''<STR_LIT>''' <EOL> v = Values ( values , comment ) <EOL> return v . to_str ( ) <EOL> def values_from_str ( values_str ) : <EOL> '''<STR_LIT>''' <EOL> v = Values ( ) <EOL> v . from_str ( values_str ) <EOL> return dict ( ( key , value ) for ( key , value ) in v . values ) <EOL> class LockService ( object ) : <EOL> '''<STR_LIT>''' <EOL> def acquire ( self , key ) : <EOL> raise NotImplementedError ( ) <EOL> def release ( self , key ) : <EOL> raise NotImplementedError ( ) <EOL> @ contextmanager <EOL> def context ( self , key ) : <EOL> yield self . acquire ( key ) <EOL> self . release ( key ) <EOL> class NoLockService ( LockService ) : <EOL> '''<STR_LIT>''' <EOL> def acquire ( self , key ) : <EOL> log . debug ( "<STR_LIT>" , key ) <EOL> pass <EOL> def release ( self , key ) : <EOL> log . debug ( "<STR_LIT>" , key ) <EOL> pass <EOL> ItemType = enum ( "<STR_LIT>" , FILE = "<STR_LIT:file>" ) <EOL> Scheme = enum ( "<STR_LIT>" , RAW = "<STR_LIT>" , LZO = "<STR_LIT>" ) <EOL> class Config ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , options = None ) : <EOL> if not options : <EOL> options = { } <EOL> self . lock_service = NoLockService ( ) <EOL> self . user = options . user if options . user else UNKNOWN_USER <EOL> self . compression = Scheme . check_value ( options . compression ) if options . compression else SCHEME_DEFAULT <EOL> self . s3_fill_keys ( options ) <EOL> self . s3_ssl = True if options . s3_ssl else False <EOL> self . no_cache = True if options . no_cache else False <EOL> def merge_in ( self , values ) : <EOL> for ( k , v ) in values . items ( ) : <EOL> log . debug ( "<STR_LIT>" , k , repr ( v ) ) <EOL> setattr ( self , k , v ) <EOL> def s3_fill_keys ( self , options ) : <EOL> '''<STR_LIT>''' <EOL> self . s3_access_key = None <EOL> self . s3_secret_key = None <EOL> keys = None <EOL> if hasattr ( options , "<STR_LIT>" ) and hasattr ( options , "<STR_LIT>" ) : <EOL> log . debug ( "<STR_LIT>" ) <EOL> keys = ( options . s3_access_key , options . s3_secret_key ) <EOL> if not keys : <EOL> keys = Config . s3_keys_from_env ( ) <EOL> if not keys and not options . no_s3cfg : <EOL> keys = Config . s3_keys_from_s3cmd ( ) <EOL> if keys : <EOL> self . s3_access_key = keys [ <NUM_LIT:0> ] <EOL> self . s3_secret_key = keys [ <NUM_LIT:1> ] <EOL> @ staticmethod <EOL> def s3_keys_from_env ( ) : <EOL> '''<STR_LIT>''' <EOL> env = os . environ <EOL> if S3_ACCESS_KEY_NAME in env and S3_SECRET_KEY_NAME in env : <EOL> keys = ( env [ S3_ACCESS_KEY_NAME ] , env [ S3_SECRET_KEY_NAME ] ) <EOL> log . debug ( "<STR_LIT>" ) <EOL> return keys <EOL> else : <EOL> return None <EOL> @ staticmethod <EOL> def s3_keys_from_s3cmd ( ) : <EOL> '''<STR_LIT>''' <EOL> s3cfg_path = None <EOL> try : <EOL> s3cfg_path = "<STR_LIT>" % os . environ [ "<STR_LIT>" ] <EOL> if not os . path . exists ( s3cfg_path ) : <EOL> return None <EOL> config = ConfigParser . ConfigParser ( ) <EOL> config . read ( s3cfg_path ) <EOL> keys = config . get ( "<STR_LIT:default>" , "<STR_LIT>" ) , config . get ( "<STR_LIT:default>" , "<STR_LIT>" ) <EOL> log . debug ( "<STR_LIT>" ) <EOL> return keys <EOL> except Exception , e : <EOL> log . info ( "<STR_LIT>" , e ) <EOL> return None <EOL> @ staticmethod <EOL> def s3store_copyfile_retries ( ) : <EOL> '''<STR_LIT>''' <EOL> return int ( os . getenv ( '<STR_LIT>' , <NUM_LIT:3> ) ) <EOL> FileState = enum ( "<STR_LIT>" , Tracked = "<STR_LIT>" , Untracked = "<STR_LIT>" , ToBeTracked = "<STR_LIT>" ) <EOL> FileStateTrackedList = [ FileState . Tracked , FileState . ToBeTracked ] <EOL> class Fingerprint ( object ) : <EOL> '''<STR_LIT>''' <EOL> DIFFER = "<STR_LIT>" <EOL> SAME = "<STR_LIT>" <EOL> UNCERTAIN = "<STR_LIT>" <EOL> EPS = <NUM_LIT> <EOL> def __init__ ( self , type = ItemType . FILE , mtime = None , size = None , md5 = None , state = FileState . Tracked , ** others ) : <EOL> self . type = type <EOL> self . mtime = mtime <EOL> self . size = size <EOL> self . md5 = md5 <EOL> self . state = state <EOL> setattrs ( self , others ) <EOL> def __repr__ ( self ) : <EOL> return self . to_str ( ) <EOL> @ staticmethod <EOL> def of_file ( local_path , md5 = None , compute_md5 = False , state = FileState . Tracked ) : <EOL> '''<STR_LIT>''' <EOL> fp = Fingerprint ( ) <EOL> fp . read_from_file ( local_path , md5 = md5 , compute_md5 = compute_md5 , state = state ) <EOL> return fp <EOL> def read_from_file ( self , path , md5 = None , compute_md5 = False , state = FileState . Tracked ) : <EOL> if not os . path . isfile ( path ) : <EOL> raise InvalidOperation ( "<STR_LIT>" % path ) <EOL> if not os . path . isfile ( path ) : <EOL> raise NotImplementedError ( "<STR_LIT>" % path ) <EOL> self . type = ItemType . FILE <EOL> self . size = os . path . getsize ( path ) <EOL> self . mtime = os . path . getmtime ( path ) <EOL> if md5 : <EOL> self . md5 = md5 <EOL> elif compute_md5 : <EOL> self . md5 = file_md5 ( path ) <EOL> else : <EOL> self . md5 = None <EOL> self . state = state <EOL> def compare ( self , fp ) : <EOL> '''<STR_LIT>''' <EOL> if self . type == ItemType . FILE and fp . type == ItemType . FILE : <EOL> if self . md5 and fp . md5 : <EOL> if self . md5 == fp . md5 : <EOL> return Fingerprint . SAME <EOL> else : <EOL> return Fingerprint . DIFFER <EOL> elif self . mtime and fp . mtime and ( self . size is not None ) and ( fp . size is not None ) : <EOL> if abs ( self . mtime - fp . mtime ) <= Fingerprint . EPS and self . size == fp . size : <EOL> return Fingerprint . SAME <EOL> elif self . size != fp . size : <EOL> return Fingerprint . DIFFER <EOL> else : <EOL> return Fingerprint . UNCERTAIN <EOL> else : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def to_str ( self ) : <EOL> return shortvals_to_str ( vars ( self ) ) <EOL> def from_str ( self , vals_str ) : <EOL> vals = shortvals_from_str ( vals_str ) <EOL> for ( k , v ) in vals . iteritems ( ) : <EOL> if k == "<STR_LIT>" : <EOL> v = float ( v ) <EOL> if k == "<STR_LIT:size>" : <EOL> v = int ( v ) <EOL> setattr ( self , k , v ) <EOL> def invalidate ( self ) : <EOL> if self . md5 : <EOL> self . md5 = INVALID_MD5 <EOL> if self . size : <EOL> self . size = INVALID_SIZE <EOL> if self . mtime : <EOL> self . mtime = INVALID_MTIME <EOL> class FileCache ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , root_path , cache_criterion ) : <EOL> '''<STR_LIT>''' <EOL> self . root_path = root_path . rstrip ( "<STR_LIT:/>" ) <EOL> self . cache_criterion = cache_criterion <EOL> self . state_path = join_path ( root_path , "<STR_LIT:state>" ) <EOL> self . contents_path = join_path ( root_path , "<STR_LIT>" ) <EOL> make_all_dirs ( self . contents_path ) <EOL> self . _read_state ( ) <EOL> def _read_state ( self ) : <EOL> self . fingerprints = FingerprintList ( ) <EOL> if os . path . isfile ( self . state_path ) : <EOL> log . debug ( "<STR_LIT>" , self . state_path ) <EOL> self . fingerprints . from_str ( values_from_str ( read_string_from_file ( self . state_path ) ) [ "<STR_LIT>" ] ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , self . state_path ) <EOL> @ log_calls <EOL> def _write_state ( self ) : <EOL> write_string_to_file ( self . state_path , <EOL> values_to_str ( [ ( "<STR_LIT>" , self . fingerprints . to_str ( ) ) ] , <EOL> comment = "<STR_LIT>" ) ) <EOL> def cache_path_for ( self , remote_path ) : <EOL> return join_path ( self . contents_path , remote_path ) <EOL> @ log_calls <EOL> def lookup ( self , remote_path ) : <EOL> '''<STR_LIT>''' <EOL> if not self . cache_criterion ( remote_path ) : <EOL> return ( None , None ) <EOL> else : <EOL> fp = self . fingerprints . lookup_fingerprint ( remote_path ) <EOL> return ( self . cache_path_for ( remote_path ) , fp ) <EOL> @ log_calls <EOL> def add ( self , remote_path , fingerprint ) : <EOL> '''<STR_LIT>''' <EOL> if not self . cache_criterion ( remote_path ) : <EOL> raise AssertionError ( "<STR_LIT>" % remote_path ) <EOL> cache_path = self . cache_path_for ( remote_path ) <EOL> log . debug ( "<STR_LIT>" , cache_path , remote_path ) <EOL> if not os . path . isfile ( cache_path ) : <EOL> raise AssertionError ( "<STR_LIT>" % ( cache_path , remote_path ) ) <EOL> self . fingerprints . update_fingerprint ( remote_path , fingerprint ) <EOL> self . _write_state ( ) <EOL> class Store ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __new__ ( cls , root_uri , config , file_cache = None ) : <EOL> '''<STR_LIT>''' <EOL> ( prefix , path ) = split_uri ( root_uri ) <EOL> for subclass in Store . __subclasses__ ( ) : <EOL> if subclass . prefix == prefix : <EOL> return super ( cls , subclass ) . __new__ ( subclass , root_uri , config , file_cache = file_cache ) <EOL> raise InvalidArgument ( "<STR_LIT>" % prefix ) <EOL> def __init__ ( self , root_uri , config , file_cache = None ) : <EOL> '''<STR_LIT>''' <EOL> self . root_uri = root_uri . rstrip ( "<STR_LIT:/>" ) <EOL> self . lock_service = config . lock_service <EOL> self . file_cache = file_cache <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % self . root_uri <EOL> @ log_calls <EOL> def read_string ( self , store_path ) : <EOL> temp_filename = new_temp_filename ( "<STR_LIT>" ) <EOL> self . download ( store_path , temp_filename ) <EOL> log . debug ( "<STR_LIT>" , temp_filename , store_path ) <EOL> with open ( temp_filename , "<STR_LIT:rb>" ) as fp : <EOL> value = fp . read ( ) <EOL> os . remove ( temp_filename ) <EOL> return value <EOL> def write_string ( self , store_path , value , clobber = False ) : <EOL> temp_filename = new_temp_filename ( "<STR_LIT>" ) <EOL> with open ( temp_filename , "<STR_LIT:wb>" ) as fp : <EOL> fp . write ( value ) <EOL> self . upload ( temp_filename , store_path , clobber = clobber ) <EOL> os . remove ( temp_filename ) <EOL> def append_string ( self , store_path , value ) : <EOL> '''<STR_LIT>''' <EOL> lock_key = join_path ( self . root_uri , store_path ) <EOL> with self . lock_service . context ( lock_key ) : <EOL> with temp_output_file ( prefix = "<STR_LIT>" ) as temp_filename : <EOL> if self . exists ( store_path ) : <EOL> self . download ( store_path , temp_filename ) <EOL> with open ( temp_filename , "<STR_LIT>" ) as fp : <EOL> fp . write ( value ) <EOL> self . upload ( temp_filename , store_path , clobber = True ) <EOL> def exists ( self , store_path ) : <EOL> '''<STR_LIT>''' <EOL> return len ( self . list ( store_path ) ) > <NUM_LIT:0> <EOL> def cache ( self , store_path , show_info = False ) : <EOL> '''<STR_LIT>''' <EOL> if not self . file_cache : <EOL> return ( None , None ) <EOL> ( cache_path , fp ) = self . file_cache . lookup ( store_path ) <EOL> if not cache_path : <EOL> return ( None , None ) <EOL> if not fp : <EOL> if os . path . exists ( cache_path ) : <EOL> log . warn ( "<STR_LIT>" , cache_path ) <EOL> fp = self . download ( store_path , cache_path , clobber = True , backup_suffix = None , show_info = show_info ) <EOL> self . file_cache . add ( store_path , fp ) <EOL> return ( cache_path , fp ) <EOL> def cached_download ( self , store_path , local_path , clobber = False , backup_suffix = None , show_info = False ) : <EOL> '''<STR_LIT>''' <EOL> if not self . file_cache : <EOL> return self . download ( store_path , local_path , clobber = clobber , backup_suffix = backup_suffix , show_info = show_info ) <EOL> if not clobber and os . path . exists ( local_path ) : <EOL> raise FileExists ( "<STR_LIT>" % local_path ) <EOL> ( cache_path , fp ) = self . cache ( store_path , show_info = show_info ) <EOL> if not cache_path : <EOL> return self . download ( store_path , local_path , clobber = clobber , backup_suffix = backup_suffix , show_info = show_info ) <EOL> if show_info : <EOL> log_fn = log . info <EOL> else : <EOL> log_fn = log . debug <EOL> log_fn ( "<STR_LIT>" , local_path , cache_path ) <EOL> copyfile_atomic ( cache_path , local_path , make_parents = True , backup_suffix = backup_suffix ) <EOL> return fp <EOL> def _log_download ( self , store_path , local_path , show_info = False ) : <EOL> if show_info : <EOL> log_fn = log . info <EOL> else : <EOL> log_fn = log . debug <EOL> log_fn ( "<STR_LIT>" , local_path , self . root_uri , store_path ) <EOL> def _log_upload ( self , local_path , store_path , show_info = False ) : <EOL> if show_info : <EOL> log_fn = log . info <EOL> else : <EOL> log_fn = log . debug <EOL> log_fn ( "<STR_LIT>" , local_path , self . root_uri , store_path ) <EOL> def list ( self , store_path ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError ( ) <EOL> def download ( self , store_path , local_path , clobber = False , backup_suffix = None , show_info = False ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError ( ) <EOL> def upload ( self , local_path , store_path , clobber = False , show_info = False ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError ( ) <EOL> class FileStore ( Store ) : <EOL> '''<STR_LIT>''' <EOL> prefix = "<STR_LIT:file>" <EOL> def __init__ ( self , root_uri , config , file_cache = None ) : <EOL> root_uri = root_uri . rstrip ( "<STR_LIT:/>" ) <EOL> Store . __init__ ( self , root_uri , config , file_cache = file_cache ) <EOL> self . root_uri = root_uri <EOL> self . root_dir = split_uri ( root_uri ) [ <NUM_LIT:1> ] <EOL> def _full_path ( self , store_path ) : <EOL> if not store_path . startswith ( "<STR_LIT:/>" ) : <EOL> store_path = "<STR_LIT:/>" + store_path <EOL> return self . root_dir + store_path ; <EOL> def list ( self , store_path ) : <EOL> full_path = self . _full_path ( store_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> store_paths = [ ] <EOL> elif os . path . isfile ( full_path ) : <EOL> store_paths = [ store_path ] <EOL> else : <EOL> store_paths = os . listdir ( self . _full_path ( store_path ) ) <EOL> return store_paths <EOL> def download ( self , store_path , local_path , clobber = False , backup_suffix = None , show_info = False ) : <EOL> self . _log_download ( store_path , local_path , show_info = show_info ) <EOL> if not clobber and os . path . exists ( local_path ) : <EOL> raise FileExists ( "<STR_LIT>" % local_path ) <EOL> copyfile_atomic ( self . _full_path ( store_path ) , local_path , make_parents = True , backup_suffix = backup_suffix ) <EOL> return Fingerprint . of_file ( local_path , compute_md5 = True ) <EOL> def upload ( self , local_path , store_path , clobber = False , show_info = False ) : <EOL> self . _log_upload ( local_path , store_path , show_info = show_info ) <EOL> target = self . _full_path ( store_path ) <EOL> if not clobber and os . path . exists ( target ) : <EOL> raise FileExists ( "<STR_LIT>" % store_path ) <EOL> copyfile_atomic ( local_path , target , make_parents = True ) <EOL> class S3Store ( Store ) : <EOL> '''<STR_LIT>''' <EOL> prefix = "<STR_LIT>" <EOL> retries = Config . s3store_copyfile_retries ( ) <EOL> def __init__ ( self , root_uri , config , file_cache = None ) : <EOL> if not config . s3_access_key or not config . s3_secret_key : <EOL> raise Failure ( "<STR_LIT>" % <EOL> ( S3_ACCESS_KEY_NAME , S3_SECRET_KEY_NAME ) ) <EOL> self . connection = boto . s3 . connection . S3Connection ( config . s3_access_key , config . s3_secret_key , is_secure = config . s3_ssl ) <EOL> root_uri = root_uri . rstrip ( "<STR_LIT:/>" ) <EOL> Store . __init__ ( self , root_uri , config , file_cache = file_cache ) <EOL> elements = split_uri ( root_uri ) [ <NUM_LIT:1> ] . split ( '<STR_LIT:/>' ) <EOL> self . root_uri = root_uri <EOL> self . root_bucket = self . connection . get_bucket ( elements [ <NUM_LIT:0> ] ) <EOL> self . root_dir = '<STR_LIT:/>' . join ( elements [ <NUM_LIT:1> : ] ) <EOL> @ log_calls <EOL> def s3_copyfile_atomic ( self , source_path , dest_path , make_parents = True , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> with atomic_output_file ( dest_path , make_parents = make_parents , backup_suffix = backup_suffix ) as tmp_path : <EOL> k = boto . s3 . key . Key ( self . root_bucket ) <EOL> key_path = join_path ( self . root_dir , source_path ) <EOL> k . key = key_path <EOL> num_attempts = <NUM_LIT:0> <EOL> md5 = None <EOL> while True : <EOL> try : <EOL> with open ( tmp_path , '<STR_LIT:wb>' ) as tmp_f : <EOL> k . get_contents_to_file ( tmp_f ) <EOL> log . debug ( "<STR_LIT>" , key_path , tmp_f ) <EOL> break <EOL> except httplib . IncompleteRead , e : <EOL> log . warn ( "<STR_LIT>" , num_attempts , e ) <EOL> tmp_k = self . root_bucket . get_key ( key_path ) <EOL> log . info ( "<STR_LIT>" , tmp_k ) <EOL> repo_md5 = tmp_k . get_metadata ( ZINC_S3_MD5_NAME ) <EOL> log . info ( "<STR_LIT>" , repo_md5 ) <EOL> if repo_md5 : <EOL> md5 = file_md5 ( tmp_path ) <EOL> log . info ( "<STR_LIT>" , md5 ) <EOL> if repo_md5 == md5 : <EOL> log . info ( "<STR_LIT>" ) <EOL> break <EOL> except Exception , e : <EOL> log . warn ( "<STR_LIT>" , source_path , e , num_attempts ) <EOL> num_attempts += <NUM_LIT:1> <EOL> if num_attempts > self . retries : <EOL> raise e <EOL> if os . path . exists ( tmp_path ) : <EOL> os . remove ( tmp_path ) <EOL> sleep_time_secs = <NUM_LIT:4> ** num_attempts <EOL> log . warn ( "<STR_LIT>" , sleep_time_secs ) <EOL> time . sleep ( sleep_time_secs ) <EOL> log . debug ( "<STR_LIT>" ) <EOL> if k . last_modified != None : <EOL> try : <EOL> modified_tuple = rfc822 . parsedate_tz ( k . last_modified ) <EOL> modified_stamp = int ( rfc822 . mktime_tz ( modified_tuple ) ) <EOL> os . utime ( tmp_path , ( modified_stamp , modified_stamp ) ) <EOL> except Exception , e : <EOL> log . warn ( "<STR_LIT>" , e ) <EOL> if md5 : <EOL> return md5 <EOL> md5 = k . get_metadata ( ZINC_S3_MD5_NAME ) <EOL> if md5 : <EOL> log . debug ( "<STR_LIT>" , k . key , md5 ) <EOL> else : <EOL> md5 = str ( k . etag ) . strip ( '<STR_LIT:">' ) <EOL> if "<STR_LIT:->" in md5 : <EOL> log . info ( "<STR_LIT>" , source_path ) <EOL> md5 = file_md5 ( tmp_path ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , k . key , md5 ) <EOL> return md5 <EOL> def list ( self , store_path ) : <EOL> rs = self . root_bucket . list ( join_path ( self . root_dir , store_path ) , delimiter = '<STR_LIT:/>' ) <EOL> return [ k . name for k in rs ] <EOL> @ log_calls <EOL> def download ( self , store_path , local_path , clobber = False , backup_suffix = None , show_info = False ) : <EOL> self . _log_download ( store_path , local_path , show_info = show_info ) <EOL> if not clobber and os . path . exists ( local_path ) : <EOL> raise FileExists ( "<STR_LIT>" % local_path ) <EOL> md5 = self . s3_copyfile_atomic ( store_path , local_path , backup_suffix = backup_suffix ) <EOL> return Fingerprint . of_file ( local_path , md5 = md5 ) <EOL> @ log_calls <EOL> def upload ( self , local_path , store_path , clobber = False , show_info = False ) : <EOL> self . _log_upload ( local_path , store_path , show_info = show_info ) <EOL> remote_path = join_path ( self . root_dir , store_path ) <EOL> if not clobber and self . exists ( remote_path ) : <EOL> raise FileExists ( "<STR_LIT>" % store_path ) <EOL> fsize = os . path . getsize ( local_path ) <EOL> local_md5 = file_md5 ( local_path ) <EOL> if fsize < S3_SINGLEPART_MAX_SIZE : <EOL> k = boto . s3 . key . Key ( self . root_bucket ) <EOL> k . key = remote_path <EOL> k . set_metadata ( ZINC_S3_MD5_NAME , local_md5 ) <EOL> k . set_contents_from_filename ( local_path ) <EOL> else : <EOL> if not hasattr ( self . root_bucket , "<STR_LIT>" ) : <EOL> raise Failure ( "<STR_LIT>" % ( fsize , local_path ) ) <EOL> mpu = self . root_bucket . initiate_multipart_upload ( remote_path , metadata = { ZINC_S3_MD5_NAME : local_md5 } ) <EOL> part = <NUM_LIT:1> <EOL> bytes_done = <NUM_LIT:0> <EOL> try : <EOL> with open ( local_path , '<STR_LIT:rb>' ) as f : <EOL> while bytes_done < fsize : <EOL> log . debug ( "<STR_LIT>" , part , local_path , remote_path ) <EOL> mpu . upload_part_from_file ( cStringIO . StringIO ( f . read ( S3_MULTIPART_CHUNK_SIZE ) ) , part ) <EOL> bytes_done += S3_MULTIPART_CHUNK_SIZE <EOL> part += <NUM_LIT:1> <EOL> cmpu = mpu . complete_upload ( ) <EOL> log . debug ( "<STR_LIT>" , local_path , remote_path , cmpu , cmpu . etag ) <EOL> except Exception , e : <EOL> mpu . cancel_upload ( ) <EOL> raise Failure ( "<STR_LIT>" % ( local_path , remote_path , e ) ) <EOL> class Item ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , path , store_path , scheme , add_scheme_suffix = True ) : <EOL> self . path = path . strip ( "<STR_LIT:/>" ) <EOL> self . store_path = store_path . strip ( "<STR_LIT:/>" ) <EOL> self . scheme = scheme <EOL> if add_scheme_suffix and self . scheme != Scheme . RAW : <EOL> assert self . scheme != "<STR_LIT>" and self . scheme . isalnum ( ) and self . scheme . islower ( ) <EOL> self . store_path += "<STR_LIT:.>" + self . scheme <EOL> if not Scheme . has_value ( self . scheme ) : <EOL> raise NotImplementedError ( "<STR_LIT>" % self . scheme ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . path , self . scheme , self . store_path ) <EOL> def display_path ( self , prefix = None ) : <EOL> return "<STR_LIT>" % ( prefix , SCOPE_SEP , self . path ) if prefix else self . path <EOL> def full_store_path ( self , repo , scope ) : <EOL> return join_path ( repo . store . root_uri , repo . store_path_dir ( scope ) , self . store_path ) <EOL> class Manifest ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , rev = None , user = UNKNOWN_USER , message = EMPTY_MESSAGE , items = None , changelist = None , time = None ) : <EOL> self . rev = rev <EOL> self . user = user <EOL> self . message = message <EOL> self . items = items if items else [ ] <EOL> self . time = time or current_repo_time ( ) <EOL> self . changelist = changelist <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . rev , len ( self . items ) ) <EOL> def to_str ( self ) : <EOL> return values_to_str ( [ ( "<STR_LIT>" , self . rev ) , <EOL> ( "<STR_LIT:user>" , self . user ) , <EOL> ( "<STR_LIT:message>" , self . message ) , <EOL> ( "<STR_LIT:time>" , self . time ) , <EOL> ( "<STR_LIT>" , self . _items_to_str ( ) ) , <EOL> ( "<STR_LIT>" , self . changelist . to_str ( ) if self . changelist else None ) <EOL> ] , comment = "<STR_LIT>" ) <EOL> def from_str ( self , mf_str ) : <EOL> values = values_from_str ( mf_str ) <EOL> self . rev = values [ "<STR_LIT>" ] <EOL> self . user = values [ "<STR_LIT:user>" ] <EOL> self . message = values [ "<STR_LIT:message>" ] <EOL> self . time = float ( values [ "<STR_LIT:time>" ] ) <EOL> self . _items_from_str ( values [ "<STR_LIT>" ] ) <EOL> if "<STR_LIT>" in values : <EOL> self . changelist = Changelist ( ) <EOL> self . changelist . from_str ( values [ "<STR_LIT>" ] ) <EOL> def to_summary ( self , verbose = True ) : <EOL> '''<STR_LIT>''' <EOL> out = [ ] <EOL> out . append ( "<STR_LIT>" % ( self . rev , format_seconds ( self . time ) , self . user , self . message ) ) <EOL> if verbose : <EOL> if self . changelist is not None : <EOL> out . append ( "<STR_LIT>" % self . changelist . to_summary ( sep = "<STR_LIT:U+002CU+0020>" ) . rstrip ( "<STR_LIT:U+002CU+0020>" ) ) <EOL> return "<STR_LIT>" . join ( out ) <EOL> def get_paths ( self ) : <EOL> return [ item . path for item in self . items ] <EOL> def get_items ( self , paths ) : <EOL> '''<STR_LIT>''' <EOL> item_dict = dict ( ( item . path , item ) for item in self . items ) <EOL> return [ ( item_dict [ p ] if p in item_dict else None ) for p in paths ] <EOL> def get_item ( self , path ) : <EOL> return self . get_items ( [ path ] ) [ <NUM_LIT:0> ] <EOL> def get_listing ( self , path , recursive = False ) : <EOL> '''<STR_LIT>''' <EOL> path = path . strip ( "<STR_LIT:/>" ) <EOL> prefix = path + "<STR_LIT:/>" if path else "<STR_LIT>" <EOL> match_rec = lambda p : ( p . startswith ( prefix ) or p == path ) <EOL> match_nonrec = lambda p : ( p . startswith ( prefix ) and ( '<STR_LIT:/>' not in p [ len ( prefix ) : ] ) or p == path ) <EOL> match_func = match_rec if recursive else match_nonrec <EOL> return [ item for item in self . items if match_func ( item . path ) ] <EOL> def _items_to_str ( self ) : <EOL> out = [ ] <EOL> for item in self . items : <EOL> type_and_scheme = "<STR_LIT>" % ( ItemType . FILE , item . scheme ) <EOL> if item . scheme == Scheme . RAW : <EOL> type_and_scheme = ItemType . FILE <EOL> out . append ( "<STR_LIT>" % ( type_and_scheme , item . path , item . store_path ) ) <EOL> return "<STR_LIT>" . join ( out ) <EOL> def _items_from_str ( self , items_str ) : <EOL> '''<STR_LIT>''' <EOL> for line in items_str . splitlines ( ) : <EOL> ( type_and_scheme , path , store_path ) = line . split ( "<STR_LIT:\t>" ) <EOL> bits = type_and_scheme . split ( "<STR_LIT:.>" ) <EOL> if bits [ <NUM_LIT:0> ] != ItemType . FILE : <EOL> raise NotImplementedError ( "<STR_LIT>" % type_and_scheme ) <EOL> if len ( bits ) == <NUM_LIT:1> : <EOL> ( type , scheme ) = ( bits [ <NUM_LIT:0> ] , Scheme . RAW ) <EOL> else : <EOL> ( type , scheme ) = ( bits [ <NUM_LIT:0> ] , bits [ <NUM_LIT:1> ] ) <EOL> self . items . append ( Item ( path , store_path , scheme , add_scheme_suffix = False ) ) <EOL> class Changelist ( object ) : <EOL> '''<STR_LIT>''' <EOL> Status = enum ( "<STR_LIT>" , ADDED = "<STR_LIT:A>" , REMOVED = "<STR_LIT:R>" , MODIFIED = "<STR_LIT:M>" ) <EOL> def __init__ ( self , add_paths = None , rm_paths = None , mod_paths = None ) : <EOL> self . add_paths = set ( add_paths ) if add_paths else set ( ) <EOL> self . rm_paths = set ( rm_paths ) if rm_paths else set ( ) <EOL> self . mod_paths = set ( mod_paths ) if mod_paths else set ( ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( hash ( self ) , self . brief_summary ( ) ) <EOL> def update ( self , path , status ) : <EOL> if status == Changelist . Status . ADDED : <EOL> self . add_paths . add ( path ) <EOL> elif status == Changelist . Status . REMOVED : <EOL> self . rm_paths . add ( path ) <EOL> elif status == Changelist . Status . MODIFIED : <EOL> self . mod_paths . add ( path ) <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % status ) <EOL> def status_dict ( self ) : <EOL> status = { } <EOL> for p in self . add_paths : <EOL> status [ p ] = Changelist . Status . ADDED <EOL> for p in self . rm_paths : <EOL> status [ p ] = Changelist . Status . REMOVED <EOL> for p in self . mod_paths : <EOL> status [ p ] = Changelist . Status . MODIFIED <EOL> return status <EOL> def all_paths ( self ) : <EOL> return self . add_paths . union ( self . rm_paths ) . union ( self . mod_paths ) <EOL> def intersect ( self , other_changelist ) : <EOL> '''<STR_LIT>''' <EOL> int_changeset = Changelist ( add_paths = self . add_paths , rm_paths = self . rm_paths , mod_paths = self . mod_paths ) <EOL> int_changeset . add_paths . intersection_update ( other_changelist . all_paths ( ) ) <EOL> int_changeset . rm_paths . intersection_update ( other_changelist . all_paths ( ) ) <EOL> int_changeset . mod_paths . intersection_update ( other_changelist . all_paths ( ) ) <EOL> return int_changeset <EOL> def difference ( self , other_changelist ) : <EOL> '''<STR_LIT>''' <EOL> int_changeset = Changelist ( add_paths = self . add_paths , rm_paths = self . rm_paths , mod_paths = self . mod_paths ) <EOL> int_changeset . add_paths . difference_update ( other_changelist . all_paths ( ) ) <EOL> int_changeset . rm_paths . difference_update ( other_changelist . all_paths ( ) ) <EOL> int_changeset . mod_paths . difference_update ( other_changelist . all_paths ( ) ) <EOL> return int_changeset <EOL> def discard ( self , path ) : <EOL> self . add_paths . discard ( path ) <EOL> self . rm_paths . discard ( path ) <EOL> self . mod_paths . discard ( path ) <EOL> def union ( self , other_changelist ) : <EOL> '''<STR_LIT>''' <EOL> uni_changeset = Changelist ( ) <EOL> uni_changeset . add_paths |= self . add_paths ^ other_changelist . add_paths <EOL> uni_changeset . rm_paths |= self . rm_paths ^ other_changelist . rm_paths <EOL> uni_changeset . rm_paths |= self . mod_paths & other_changelist . rm_paths <EOL> uni_changeset . mod_paths |= self . add_paths & other_changelist . mod_paths <EOL> uni_changeset . mod_paths |= self . rm_paths & other_changelist . add_paths <EOL> uni_changeset . mod_paths |= self . mod_paths | other_changelist . mod_paths <EOL> return uni_changeset <EOL> def is_empty ( self ) : <EOL> return len ( self . add_paths ) == <NUM_LIT:0> and len ( self . rm_paths ) == <NUM_LIT:0> and len ( self . mod_paths ) == <NUM_LIT:0> <EOL> def brief_summary ( self ) : <EOL> if len ( self . add_paths ) + len ( self . rm_paths ) + len ( self . mod_paths ) == <NUM_LIT:0> : <EOL> return "<STR_LIT>" <EOL> else : <EOL> return "<STR_LIT>" % ( len ( self . add_paths ) , len ( self . rm_paths ) , len ( self . mod_paths ) ) <EOL> def to_summary ( self , prefix = None , sep = "<STR_LIT:\n>" ) : <EOL> '''<STR_LIT>''' <EOL> status = self . status_dict ( ) <EOL> out = [ ] <EOL> prefix = prefix + SCOPE_SEP if prefix else "<STR_LIT>" <EOL> for p in sorted ( status ) : <EOL> out . append ( "<STR_LIT>" % ( status [ p ] , prefix , p , sep ) ) <EOL> return "<STR_LIT>" . join ( out ) <EOL> def to_str ( self ) : <EOL> status = self . status_dict ( ) <EOL> out = [ ] <EOL> for p in sorted ( status ) : <EOL> out . append ( "<STR_LIT>" % ( status [ p ] , p ) ) <EOL> return "<STR_LIT>" . join ( out ) <EOL> @ log_calls <EOL> def from_str ( self , changelist_str ) : <EOL> self . add_paths = [ ] <EOL> self . rm_paths = [ ] <EOL> self . mod_paths = [ ] <EOL> for line in changelist_str . splitlines ( ) : <EOL> ( status , path ) = line . split ( "<STR_LIT:\t>" ) <EOL> if status == Changelist . Status . ADDED : <EOL> self . add_paths . append ( path ) <EOL> elif status == Changelist . Status . REMOVED : <EOL> self . rm_paths . append ( path ) <EOL> elif status == Changelist . Status . MODIFIED : <EOL> self . mod_paths . append ( path ) <EOL> else : <EOL> raise InvalidArgument ( "<STR_LIT>" % status ) <EOL> def compress_lzo ( src_path , target_path ) : <EOL> with atomic_output_file ( target_path , make_parents = True ) as temp_target_path : <EOL> shell_command ( [ "<STR_LIT>" , "<STR_LIT>" % LZO_LEVEL , "<STR_LIT>" , temp_target_path , src_path ] ) <EOL> def decompress_lzo ( src_path , target_path ) : <EOL> with atomic_output_file ( target_path , make_parents = True ) as temp_target_path : <EOL> shell_command ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , temp_target_path , src_path ] ) <EOL> def compress_file ( src_path , target_path , scheme ) : <EOL> log . debug ( "<STR_LIT>" , src_path , target_path , scheme ) <EOL> if scheme == Scheme . RAW : <EOL> copyfile_atomic ( src_path , target_path ) <EOL> elif scheme == Scheme . LZO : <EOL> compress_lzo ( src_path , target_path ) <EOL> else : <EOL> raise NotImplementedError ( "<STR_LIT>" % scheme ) <EOL> def decompress_file ( src_path , target_path , scheme ) : <EOL> if scheme == Scheme . RAW : <EOL> copyfile_atomic ( src_path , target_path ) <EOL> elif scheme == Scheme . LZO : <EOL> decompress_lzo ( src_path , target_path ) <EOL> else : <EOL> raise NotImplementedError ( "<STR_LIT>" % scheme ) <EOL> fp = Fingerprint . of_file ( target_path , compute_md5 = True ) <EOL> return fp <EOL> @ contextmanager <EOL> def compressed_local_file ( local_path , scheme ) : <EOL> '''<STR_LIT>''' <EOL> if scheme == Scheme . RAW : <EOL> yield local_path <EOL> else : <EOL> compressed_path = new_temp_filename ( prefix = "<STR_LIT>" , based_on = local_path ) <EOL> compress_file ( local_path , compressed_path , scheme ) <EOL> yield compressed_path <EOL> os . remove ( compressed_path ) <EOL> @ contextmanager <EOL> def decompressed_local_file ( local_path , scheme ) : <EOL> '''<STR_LIT>''' <EOL> if scheme == Scheme . RAW : <EOL> yield local_path <EOL> else : <EOL> decompressed_path = new_temp_filename ( prefix = "<STR_LIT>" , based_on = local_path ) <EOL> decompress_file ( local_path , decompressed_path , scheme ) <EOL> yield decompressed_path <EOL> os . remove ( decompressed_path ) <EOL> class Fetcher ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , store ) : <EOL> self . store = store <EOL> def fetch_item ( self , scope , item , local_path , target_is_dir = False , clobber = False , backup_suffix = BACKUP_SUFFIX , show_info = True ) : <EOL> remote_dir = Repo . store_path_dir ( scope ) <EOL> remote_path = join_path ( remote_dir , item . store_path ) <EOL> if target_is_dir : <EOL> local_path = join_path ( local_path , item . path ) <EOL> if item . scheme == Scheme . RAW : <EOL> fp = self . store . cached_download ( remote_path , local_path , clobber = clobber , backup_suffix = backup_suffix , show_info = True ) <EOL> else : <EOL> with temp_output_file ( prefix = "<STR_LIT>" , based_on = local_path ) as compressed_path : <EOL> self . store . cached_download ( remote_path , compressed_path , clobber = clobber , backup_suffix = backup_suffix , show_info = True ) <EOL> fp = decompress_file ( compressed_path , local_path , item . scheme ) <EOL> return fp <EOL> def store_item ( self , scope , item , local_dir , show_info = True ) : <EOL> local_path = join_path ( local_dir , item . path ) <EOL> store_path = join_path ( Repo . store_path_dir ( scope ) , item . store_path ) <EOL> with compressed_local_file ( local_path , item . scheme ) as compressed_path : <EOL> self . store . upload ( compressed_path , store_path , show_info = show_info ) <EOL> class Repo ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , root_uri , config , check_valid = True , file_cache = None ) : <EOL> '''<STR_LIT>''' <EOL> log . debug ( "<STR_LIT>" , root_uri ) <EOL> self . store = Store ( root_uri , config , file_cache = file_cache ) <EOL> self . fetcher = Fetcher ( self . store ) <EOL> self . config = config <EOL> if check_valid : <EOL> self . check_root ( ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % self . store <EOL> @ log_calls <EOL> def check_root ( self ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> version_info = values_from_str ( self . store . read_string ( self . _store_path_version ( ) ) ) <EOL> found_repo_version = version_info [ "<STR_LIT>" ] <EOL> if found_repo_version not in REPO_VERSIONS_SUPPORTED : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( REPO_VERSIONS_SUPPORTED , found_repo_version , self . store . root_uri ) ) <EOL> except Exception , e : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( self . store . root_uri , REPO_VERSION , str ( e ) . strip ( ) ) ) <EOL> log . debug ( "<STR_LIT>" , found_repo_version ) <EOL> @ contextmanager <EOL> def scope_lock ( self , scope ) : <EOL> '''<STR_LIT>''' <EOL> lock_key = join_path ( self . store . root_uri , self . store_path_dir ( scope ) ) <EOL> with self . config . lock_service . context ( lock_key ) as value : <EOL> yield value <EOL> @ log_calls <EOL> def init_repo ( self ) : <EOL> '''<STR_LIT>''' <EOL> version_str = values_to_str ( { "<STR_LIT>" : REPO_VERSION } ) <EOL> self . store . write_string ( self . _store_path_version ( ) , version_str ) <EOL> @ log_calls <EOL> def create_scope ( self , path , user = None , message = None , time = None ) : <EOL> '''<STR_LIT>''' <EOL> if not user : <EOL> user = UNKNOWN_USER <EOL> if not message : <EOL> message = "<STR_LIT>" % path <EOL> if self . get_tip ( path , expect = False ) is not None : <EOL> raise InvalidOperation ( "<STR_LIT>" % path , suppressable = True ) <EOL> rev = self . _commit_items ( path , None , [ ] , Changelist ( ) , user = user , message = message , time = time ) <EOL> self . store . append_string ( self . _store_path_scopes ( ) , path + "<STR_LIT:\n>" ) <EOL> log . info ( "<STR_LIT>" , path , rev ) <EOL> def get_scopes ( self ) : <EOL> '''<STR_LIT>''' <EOL> scopes_file = self . _store_path_scopes ( ) <EOL> if not self . store . exists ( scopes_file ) : <EOL> raise InvalidOperation ( "<STR_LIT>" , scopes_file ) <EOL> scopes = sorted ( self . _parse_scopes ( self . store . read_string ( self . _store_path_scopes ( ) ) ) ) <EOL> return scopes <EOL> def get_revs ( self , scope , expect = True ) : <EOL> '''<STR_LIT>''' <EOL> commits_file = self . _store_path_commits ( scope ) <EOL> if not self . store . exists ( commits_file ) : <EOL> log . debug ( "<STR_LIT>" , commits_file ) <EOL> return None <EOL> revs = self . _parse_commits ( self . store . read_string ( self . _store_path_commits ( scope ) ) ) <EOL> return revs <EOL> @ log_calls <EOL> def get_tags ( self , scope , srev_start = None , srev_end = None , filter = None ) : <EOL> '''<STR_LIT>''' <EOL> tags_file = self . _store_path_tags ( scope ) <EOL> if not self . store . exists ( tags_file ) : <EOL> log . debug ( "<STR_LIT>" , tags_file ) <EOL> return <EOL> all_tags = self . _parse_tags ( self . store . read_string ( self . _store_path_tags ( scope ) ) ) <EOL> tag_dict = group_pairs ( ( rev , name ) for ( name , rev ) in all_tags ) <EOL> revs = self . _rev_range ( scope , srev_start , srev_end ) <EOL> for rev in revs : <EOL> if rev in tag_dict : <EOL> for tag in sorted ( tag_dict [ rev ] ) : <EOL> if not filter or filter ( self . get_manifest ( scope , rev ) ) : <EOL> yield ( tag , rev ) <EOL> def _add_tag ( self , scope , tag_name , rev ) : <EOL> '''<STR_LIT>''' <EOL> scope = scope . strip ( "<STR_LIT:/>" ) <EOL> self . store . append_string ( self . _store_path_tags ( scope ) , rev + "<STR_LIT:\t>" + tag_name + "<STR_LIT:\n>" ) <EOL> @ staticmethod <EOL> def _parse_tags ( tags_str ) : <EOL> tags = [ tuple ( line . split ( "<STR_LIT:\t>" ) ) for line in tags_str . splitlines ( ) ] <EOL> return [ ( name , rev ) for ( rev , name ) in tags ] <EOL> def get_tip ( self , scope , expect = True ) : <EOL> '''<STR_LIT>''' <EOL> revs = self . get_revs ( scope ) <EOL> if revs : <EOL> tip = revs [ - <NUM_LIT:1> ] <EOL> else : <EOL> tip = None <EOL> if expect and tip is None : <EOL> raise InvalidArgument ( "<STR_LIT>" % scope ) <EOL> return tip <EOL> @ log_calls <EOL> def get_manifest ( self , scope , srev ) : <EOL> '''<STR_LIT>''' <EOL> rev = self . resolve_rev ( scope , srev ) <EOL> mf = Manifest ( ) <EOL> mf . from_str ( self . store . read_string ( self . _store_path_manifest ( scope , rev ) ) ) <EOL> return mf <EOL> @ log_calls <EOL> def fetch_items ( self , scope , items , local_dir , clobber = True , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> out = { } <EOL> with temp_output_dir ( local_dir , backup_suffix = backup_suffix ) as temp_dir : <EOL> for item in items : <EOL> fp = self . fetcher . fetch_item ( scope , item , temp_dir , target_is_dir = True , clobber = clobber , backup_suffix = backup_suffix , show_info = True ) <EOL> out [ item . path ] = fp <EOL> return FingerprintList ( fingerprints = out ) <EOL> @ log_calls <EOL> def copy ( self , scope , path , srev , local_path , clobber = False , recursive = False , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> mf = self . get_manifest ( scope , srev ) <EOL> if recursive : <EOL> items = mf . get_listing ( path , recursive = True ) <EOL> if not items : <EOL> raise Failure ( "<STR_LIT>" % ( path , mf . rev , scope ) , suppressable = True ) <EOL> self . fetch_items ( scope , items , local_path , clobber = clobber ) <EOL> else : <EOL> if os . path . isdir ( local_path ) : <EOL> local_path = join_path ( local_path , os . path . basename ( path ) ) <EOL> item = mf . get_item ( path ) <EOL> if not item : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( path , mf . rev , scope ) ) <EOL> self . fetcher . fetch_item ( scope , item , local_path , target_is_dir = False , clobber = clobber , backup_suffix = backup_suffix , show_info = True ) <EOL> return mf . rev <EOL> @ log_calls <EOL> def list ( self , scope , path , srev , recursive = False ) : <EOL> '''<STR_LIT>''' <EOL> mf = self . get_manifest ( scope , srev ) <EOL> items = mf . get_listing ( path , recursive = recursive ) <EOL> if not items : <EOL> raise Failure ( "<STR_LIT>" % ( path , srev , scope ) , suppressable = True ) <EOL> return items <EOL> @ log_calls <EOL> def locate ( self , scope , path , srev , recursive = False ) : <EOL> '''<STR_LIT>''' <EOL> items = self . list ( scope , path , srev , recursive = recursive ) <EOL> assert len ( items ) >= <NUM_LIT:1> <EOL> return [ ( item . scheme , item . full_store_path ( self , scope ) ) for item in items ] <EOL> @ log_calls <EOL> def commit ( self , scope , local_dir , prev_rev , changelist , user = UNKNOWN_USER , message = EMPTY_MESSAGE , time = None ) : <EOL> '''<STR_LIT>''' <EOL> with self . scope_lock ( scope ) : <EOL> tip_rev = self . get_tip ( scope ) <EOL> if prev_rev != tip_rev : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( prev_rev , tip_rev , scope ) ) <EOL> prev_mf = self . get_manifest ( scope , prev_rev ) <EOL> return self . _commit_items ( scope , local_dir , prev_mf . items , changelist , user , message , time = time ) <EOL> @ log_calls <EOL> def _commit_items ( self , scope , local_dir , prev_items , changelist , user = UNKNOWN_USER , message = EMPTY_MESSAGE , time = None ) : <EOL> new_rev = self . _new_rev ( scope ) <EOL> mf = Manifest ( rev = new_rev , user = user , message = message , changelist = changelist , time = time ) <EOL> ( new_items , changed_items ) = self . _apply_changelist_to_items ( prev_items , changelist , new_rev ) <EOL> mf . items = new_items <EOL> new_paths = [ item . path for item in changed_items ] <EOL> self . _commit_from_manifest ( scope , mf , new_paths , local_dir ) <EOL> return new_rev <EOL> def _rev_range ( self , scope , srev_start = None , srev_end = None ) : <EOL> '''<STR_LIT>''' <EOL> rev_start = self . resolve_rev ( scope , srev_start ) if srev_start else srev_start <EOL> rev_end = self . resolve_rev ( scope , srev_end ) if srev_end else srev_end <EOL> all_revs = self . get_revs ( scope ) <EOL> if not all_revs : <EOL> raise InvalidArgument ( "<STR_LIT>" % scope ) <EOL> if not rev_start and not rev_end : <EOL> rev_start_index = len ( all_revs ) - <NUM_LIT:1> <EOL> rev_end_index = <NUM_LIT:0> <EOL> if rev_start : <EOL> try : <EOL> rev_start_index = all_revs . index ( rev_start ) <EOL> except ValueError , e : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( rev_start , scope ) ) <EOL> if not rev_end : <EOL> rev_end_index = <NUM_LIT:0> <EOL> if rev_end : <EOL> try : <EOL> rev_end_index = all_revs . index ( rev_end ) <EOL> except ValueError , e : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( rev_end , scope ) ) <EOL> if not rev_start : <EOL> rev_start_index = <NUM_LIT:0> <EOL> log . debug ( "<STR_LIT>" , rev_start , rev_end , rev_start_index , rev_end_index , len ( all_revs ) ) <EOL> if rev_start_index <= rev_end_index : <EOL> revs = all_revs [ rev_start_index : rev_end_index + <NUM_LIT:1> ] <EOL> else : <EOL> revs = reversed ( all_revs [ rev_end_index : rev_start_index + <NUM_LIT:1> ] ) <EOL> return revs <EOL> @ log_calls <EOL> def log ( self , scope , stream , srev_start = None , srev_end = None , filter = None ) : <EOL> '''<STR_LIT>''' <EOL> revs = self . _rev_range ( scope , srev_start , srev_end ) <EOL> for rev in revs : <EOL> mf = self . get_manifest ( scope , rev ) <EOL> if filter and not filter ( mf ) : <EOL> continue <EOL> if not mf : <EOL> log . warn ( "<STR_LIT>" , rev ) <EOL> continue <EOL> stream . write ( mf . to_summary ( ) ) <EOL> stream . write ( "<STR_LIT:\n>" ) <EOL> @ log_calls <EOL> def tag ( self , scope , tag_name , srev = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> rev = self . resolve_rev ( scope , srev ) <EOL> self . _add_tag ( scope , tag_name , rev ) <EOL> @ log_calls <EOL> def _apply_changelist_to_items ( self , items , changelist , new_rev ) : <EOL> '''<STR_LIT>''' <EOL> all_paths = set ( [ item . path for item in items ] ) <EOL> assert changelist . rm_paths . issubset ( all_paths ) <EOL> assert changelist . mod_paths . issubset ( all_paths ) <EOL> assert len ( changelist . add_paths . intersection ( all_paths ) ) == <NUM_LIT:0> <EOL> new_items = [ ] <EOL> changed_items = [ ] <EOL> for item in items : <EOL> if item . path in changelist . rm_paths : <EOL> log . debug ( "<STR_LIT>" , item ) <EOL> pass <EOL> elif item . path in changelist . mod_paths : <EOL> new_item = Item ( item . path , self . _store_path_file ( item . path , new_rev ) , self . config . compression ) <EOL> log . debug ( "<STR_LIT>" , new_item ) <EOL> new_items . append ( new_item ) <EOL> changed_items . append ( new_item ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , item ) <EOL> new_items . append ( item ) <EOL> for path in changelist . add_paths : <EOL> new_item = Item ( path , self . _store_path_file ( path , new_rev ) , self . config . compression ) <EOL> log . debug ( "<STR_LIT>" , new_item ) <EOL> new_items . append ( new_item ) <EOL> changed_items . append ( new_item ) <EOL> new_items . sort ( key = lambda item : item . path ) <EOL> changed_items . sort ( key = lambda item : item . path ) <EOL> return ( new_items , changed_items ) <EOL> def _commit_from_manifest ( self , scope , mf , new_paths , local_dir ) : <EOL> '''<STR_LIT>''' <EOL> new_paths = set ( new_paths ) <EOL> for item in mf . items : <EOL> if item . path in new_paths : <EOL> self . fetcher . store_item ( scope , item , local_dir , show_info = True ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , item ) <EOL> self . store . write_string ( self . _store_path_manifest ( scope , mf . rev ) , mf . to_str ( ) ) <EOL> self . _add_commit ( scope , mf . rev ) <EOL> def _add_commit ( self , scope , rev ) : <EOL> '''<STR_LIT>''' <EOL> scope = scope . strip ( "<STR_LIT:/>" ) <EOL> self . store . append_string ( self . _store_path_commits ( scope ) , rev + "<STR_LIT:\n>" ) <EOL> @ log_calls <EOL> def resolve_rev ( self , scope , srev , expect = True ) : <EOL> '''<STR_LIT>''' <EOL> if Repo . _is_rev ( srev ) : <EOL> return srev <EOL> if srev == "<STR_LIT>" : <EOL> rev = self . get_tip ( scope ) <EOL> else : <EOL> rev = srev <EOL> tags = dict ( self . get_tags ( scope ) ) <EOL> if tags and srev in tags : <EOL> rev = tags [ srev ] <EOL> log . debug ( "<STR_LIT>" , srev , rev ) <EOL> if expect and not rev : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( srev , scope ) ) <EOL> return rev <EOL> @ staticmethod <EOL> def _new_rev ( scope ) : <EOL> '''<STR_LIT>''' <EOL> return new_uid ( REV_BITS ) <EOL> REV_RE = re . compile ( "<STR_LIT>" % len ( new_uid ( REV_BITS ) ) ) <EOL> @ staticmethod <EOL> def _is_rev ( srev ) : <EOL> '''<STR_LIT>''' <EOL> return Repo . REV_RE . match ( srev ) is not None <EOL> @ staticmethod <EOL> def _parse_commits ( commits_str ) : <EOL> return commits_str . splitlines ( ) <EOL> @ staticmethod <EOL> def _parse_scopes ( scopes_str ) : <EOL> return scopes_str . splitlines ( ) <EOL> @ staticmethod <EOL> def store_path_dir ( path ) : <EOL> '''<STR_LIT>''' <EOL> if path == "<STR_LIT>" : <EOL> return "<STR_LIT>" <EOL> else : <EOL> path = path . strip ( "<STR_LIT:/>" ) <EOL> elements = path . split ( "<STR_LIT:/>" ) <EOL> return "<STR_LIT>" . join ( [ "<STR_LIT>" + p for p in elements ] ) . lstrip ( "<STR_LIT:/>" ) <EOL> @ staticmethod <EOL> def _store_path_commits ( scope , branch = "<STR_LIT>" ) : <EOL> '''<STR_LIT>''' <EOL> return ( "<STR_LIT>" % ( Repo . store_path_dir ( scope ) , branch ) ) . lstrip ( "<STR_LIT:/>" ) <EOL> @ staticmethod <EOL> def _store_path_tags ( scope ) : <EOL> '''<STR_LIT>''' <EOL> return ( Repo . store_path_dir ( scope ) + "<STR_LIT>" ) . lstrip ( "<STR_LIT:/>" ) <EOL> @ staticmethod <EOL> def _store_path_file ( path , rev ) : <EOL> '''<STR_LIT>''' <EOL> path = path . strip ( "<STR_LIT:/>" ) <EOL> elements = path . split ( "<STR_LIT:/>" ) <EOL> filename = elements [ - <NUM_LIT:1> ] <EOL> dirname = "<STR_LIT:/>" . join ( elements [ : - <NUM_LIT:1> ] ) <EOL> return ( "<STR_LIT>" % ( Repo . store_path_dir ( dirname ) , rev , filename ) ) . lstrip ( "<STR_LIT:/>" ) <EOL> @ staticmethod <EOL> def _store_path_manifest ( scope , rev ) : <EOL> return ( "<STR_LIT>" % ( Repo . store_path_dir ( scope ) , rev ) ) . lstrip ( "<STR_LIT:/>" ) <EOL> @ staticmethod <EOL> def _store_path_scopes ( ) : <EOL> return "<STR_LIT>" <EOL> @ staticmethod <EOL> def _store_path_version ( ) : <EOL> return "<STR_LIT>" <EOL> @ staticmethod <EOL> def store_path_is_cachable ( path ) : <EOL> ( kind , _parents , _details ) = Repo . decompose_store_path ( path ) <EOL> return kind == "<STR_LIT>" or kind == "<STR_LIT:file>" <EOL> @ staticmethod <EOL> def decompose_store_path ( path ) : <EOL> '''<STR_LIT>''' <EOL> def classify ( pieces , parents ) : <EOL> head = pieces [ <NUM_LIT:0> ] <EOL> if head in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> assert len ( pieces ) == <NUM_LIT:1> <EOL> return ( "<STR_LIT>" , parents , head ) <EOL> elif head == '<STR_LIT>' : <EOL> assert len ( pieces ) == <NUM_LIT:1> <EOL> return ( "<STR_LIT>" , parents , "<STR_LIT>" ) <EOL> elif head == '<STR_LIT>' : <EOL> assert len ( pieces ) == <NUM_LIT:3> <EOL> return ( "<STR_LIT>" , parents , pieces [ <NUM_LIT:1> : ] ) <EOL> elif head == '<STR_LIT>' : <EOL> assert len ( pieces ) == <NUM_LIT:3> <EOL> return ( "<STR_LIT>" , parents , pieces [ <NUM_LIT:1> : ] ) <EOL> elif head == '<STR_LIT>' : <EOL> assert len ( pieces ) == <NUM_LIT:3> <EOL> return ( "<STR_LIT:file>" , parents + [ pieces [ - <NUM_LIT:1> ] ] , pieces [ <NUM_LIT:1> : ] ) <EOL> elif head == '<STR_LIT>' : <EOL> return classify ( pieces [ <NUM_LIT:2> : ] , parents + [ pieces [ <NUM_LIT:1> ] ] ) <EOL> else : <EOL> raise AssertionError ( "<STR_LIT>" % pieces ) <EOL> ( kind , parents , details ) = classify ( path . strip ( "<STR_LIT:/>" ) . split ( "<STR_LIT:/>" ) , [ ] ) <EOL> parents = join_path ( * parents ) <EOL> if type ( details ) == list : <EOL> details = join_path ( * details ) <EOL> return ( kind , parents , details ) <EOL> Mode = enum ( "<STR_LIT>" , AUTO = "<STR_LIT>" , PARTIAL = "<STR_LIT>" ) <EOL> class Checkout ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , scope , rev , mode , ** others ) : <EOL> self . scope = scope <EOL> self . rev = rev <EOL> self . mode = mode <EOL> setattrs ( self , others ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . scope , self . rev , self . mode ) <EOL> class CheckoutList ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , checkout_list = None ) : <EOL> if not checkout_list : <EOL> checkout_list = [ ] <EOL> self . clear ( ) <EOL> for checkout in checkout_list : <EOL> self . add ( checkout ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % len ( self . checkouts ) <EOL> def clear ( self ) : <EOL> self . checkouts = { } <EOL> def add ( self , checkout ) : <EOL> self . checkouts [ checkout . scope ] = checkout <EOL> def has_scope ( self , scope ) : <EOL> return scope in self . checkouts <EOL> def update_rev ( self , scope , rev ) : <EOL> if self . has_scope ( scope ) : <EOL> self . checkouts [ scope ] . rev = rev <EOL> else : <EOL> self . checkouts [ scope ] = Checkout ( scope , rev , Mode . AUTO ) <EOL> def rev ( self , scope ) : <EOL> return self . checkouts [ scope ] . rev if scope in self . checkouts else None <EOL> def as_list ( self ) : <EOL> return sorted ( list ( self . checkouts . itervalues ( ) ) , key = ( lambda c : c . scope ) ) <EOL> def to_str ( self ) : <EOL> out = [ ] <EOL> for scope in sorted ( self . checkouts . keys ( ) ) : <EOL> out . append ( shortvals_to_str ( vars ( self . checkouts [ scope ] ) ) ) <EOL> return "<STR_LIT:\n>" . join ( out ) <EOL> def from_str ( self , checkouts_str ) : <EOL> self . clear ( ) <EOL> for line in checkouts_str . splitlines ( ) : <EOL> line = line . strip ( ) <EOL> if line and not line . startswith ( "<STR_LIT:#>" ) : <EOL> if "<STR_LIT:\t>" in line : <EOL> ( scope , rev ) = line . split ( "<STR_LIT:\t>" ) <EOL> self . checkouts [ scope ] = Checkout ( scope , rev , Mode . AUTO ) <EOL> else : <EOL> init_property = shortvals_from_str ( line ) <EOL> self . checkouts [ init_property [ "<STR_LIT>" ] ] = Checkout ( ** init_property ) <EOL> class FingerprintList ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , fingerprints = None ) : <EOL> '''<STR_LIT>''' <EOL> self . clear ( ) <EOL> if fingerprints : <EOL> self . fingerprints = fingerprints <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % len ( self . fingerprints ) <EOL> def clear ( self ) : <EOL> self . fingerprints = { } <EOL> def lookup_fingerprint ( self , local_path ) : <EOL> return self . fingerprints [ local_path ] if local_path in self . fingerprints else None <EOL> def update_fingerprint ( self , local_path , fingerprint ) : <EOL> self . fingerprints [ local_path ] = fingerprint <EOL> def fill_md5s ( self , other_fingerprints ) : <EOL> '''<STR_LIT>''' <EOL> count = <NUM_LIT:0> <EOL> other_fps = other_fingerprints . fingerprints <EOL> for ( path , fp ) in self . fingerprints . iteritems ( ) : <EOL> if path in other_fps : <EOL> other_fp = other_fps [ path ] <EOL> if other_fp . md5 and fp . compare ( other_fp ) == Fingerprint . SAME : <EOL> fp . md5 = other_fp . md5 <EOL> count = count + <NUM_LIT:1> <EOL> log . debug ( "<STR_LIT>" , count , len ( self . fingerprints ) ) <EOL> return count <EOL> def compute_md5s ( self , root_path ) : <EOL> '''<STR_LIT>''' <EOL> count = <NUM_LIT:0> <EOL> for ( path , fp ) in self . fingerprints . iteritems ( ) : <EOL> if not fp . md5 : <EOL> full_path = join_path ( root_path , path ) <EOL> log . debug ( "<STR_LIT>" , full_path ) <EOL> new_fp = Fingerprint . of_file ( full_path , compute_md5 = True ) <EOL> if new_fp . compare ( fp ) == Fingerprint . SAME : <EOL> fp . md5 = new_fp . md5 <EOL> fp . mtime = new_fp . mtime <EOL> else : <EOL> raise BadState ( "<STR_LIT>" % ( new_fp , fp , full_path ) ) <EOL> count = count + <NUM_LIT:1> <EOL> log . debug ( "<STR_LIT>" , count , len ( self . fingerprints ) ) <EOL> return count <EOL> @ staticmethod <EOL> def of_files ( root_path , path_list , compute_md5 = False , state = FileState . Tracked ) : <EOL> '''<STR_LIT>''' <EOL> fingerprints = FingerprintList ( ) <EOL> for path in path_list : <EOL> full_path = join_path ( root_path , path ) <EOL> fp = Fingerprint . of_file ( full_path , compute_md5 = compute_md5 , state = state ) <EOL> fingerprints . update_fingerprint ( path , fp ) <EOL> return fingerprints <EOL> def to_str ( self ) : <EOL> out = [ ] <EOL> for path in sorted ( self . fingerprints . keys ( ) ) : <EOL> fp = self . fingerprints [ path ] <EOL> out . append ( "<STR_LIT>" % ( path , fp ) ) <EOL> return "<STR_LIT>" . join ( out ) <EOL> def from_str ( self , checkout_state_str ) : <EOL> self . clear ( ) <EOL> for line in checkout_state_str . splitlines ( ) : <EOL> line = line . strip ( ) <EOL> if line and not line . startswith ( "<STR_LIT:#>" ) : <EOL> ( path , fingerprint_str ) = line . split ( "<STR_LIT:\t>" ) <EOL> fp = Fingerprint ( ) <EOL> fp . from_str ( fingerprint_str ) <EOL> self . update_fingerprint ( path , fp ) <EOL> class CheckoutState ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , work_dir , checkouts = None , fingerprints = None ) : <EOL> if not checkouts : <EOL> checkouts = CheckoutList ( ) <EOL> if not fingerprints : <EOL> fingerprints = FingerprintList ( ) <EOL> self . work_dir = work_dir <EOL> self . checkouts = checkouts <EOL> self . fingerprints = fingerprints <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . work_dir , len ( self . checkouts . checkouts ) , len ( self . fingerprints . fingerprints ) ) <EOL> def _check_file_exists ( self , local_path ) : <EOL> full_path = join_path ( self . work_dir , local_path ) <EOL> if not os . path . isfile ( full_path ) : <EOL> if os . path . exists ( full_path ) : <EOL> raise InvalidOperation ( "<STR_LIT>" % full_path ) <EOL> else : <EOL> raise InvalidOperation ( "<STR_LIT>" % full_path ) <EOL> def has_scope ( self , scope ) : <EOL> return self . checkouts . has_scope ( scope ) <EOL> def scope_mode ( self , scope , mode = None ) : <EOL> if mode is None : <EOL> return self . checkouts . checkouts [ scope ] . mode if self . has_scope ( scope ) else None <EOL> else : <EOL> if self . has_scope ( scope ) : <EOL> self . checkouts . checkouts [ scope ] . mode = mode <EOL> else : <EOL> raise InvalidOperation ( "<STR_LIT>" % scope ) <EOL> def for_subdir ( self , scope ) : <EOL> '''<STR_LIT>''' <EOL> fingerprints = FingerprintList ( ) <EOL> prefix = scope . strip ( "<STR_LIT:/>" ) + "<STR_LIT:/>" <EOL> for ( local_path , fp ) in self . fingerprints . fingerprints . iteritems ( ) : <EOL> if local_path . startswith ( prefix ) : <EOL> if fp . state == FileState . Tracked : <EOL> fingerprints . update_fingerprint ( local_path [ len ( prefix ) : ] , fp ) <EOL> elif fp . state == FileState . ToBeTracked and self . scope_mode ( scope ) == Mode . PARTIAL : <EOL> full_path = join_path ( self . work_dir , local_path ) <EOL> if not os . path . exists ( full_path ) : <EOL> log . warn ( "<STR_LIT>" , full_path ) <EOL> return fingerprints <EOL> def update_checkout_rev ( self , scope , rev ) : <EOL> self . checkouts . update_rev ( scope , rev ) <EOL> def update_item ( self , scope , path , fingerprint ) : <EOL> local_path = join_path ( scope , path ) <EOL> self . _check_file_exists ( local_path ) <EOL> self . fingerprints . update_fingerprint ( local_path , fingerprint ) <EOL> def update_fingerprints ( self , scope , fingerprints ) : <EOL> for ( path , fp ) in fingerprints . fingerprints . iteritems ( ) : <EOL> self . update_item ( scope , path , fp ) <EOL> def update_from_changelist ( self , scope , changelist , new_fingerprints ) : <EOL> '''<STR_LIT>''' <EOL> for path in changelist . add_paths : <EOL> self . update_item ( scope , path , new_fingerprints . fingerprints [ path ] ) <EOL> for path in changelist . mod_paths : <EOL> self . update_item ( scope , path , new_fingerprints . fingerprints [ path ] ) <EOL> for path in changelist . rm_paths : <EOL> self . delete_item ( scope , path ) <EOL> def delete_item ( self , scope , path ) : <EOL> local_path = join_path ( scope , path ) <EOL> self . fingerprints . fingerprints . pop ( local_path ) <EOL> def to_str ( self ) : <EOL> for ( path , fp ) in self . fingerprints . fingerprints . iteritems ( ) : <EOL> if not fp or not fp . md5 : <EOL> raise AssertionError ( "<STR_LIT>" % ( path , fp ) ) <EOL> return values_to_str ( [ ( "<STR_LIT>" , self . checkouts . to_str ( ) ) , <EOL> ( "<STR_LIT>" , self . fingerprints . to_str ( ) ) ] , <EOL> comment = "<STR_LIT>" ) <EOL> def from_str ( self , checkout_state_str ) : <EOL> values = values_from_str ( checkout_state_str ) <EOL> self . checkouts = CheckoutList ( ) <EOL> self . checkouts . from_str ( values [ "<STR_LIT>" ] ) <EOL> self . fingerprints = FingerprintList ( ) <EOL> self . fingerprints . from_str ( values [ "<STR_LIT>" ] ) <EOL> def changelist_for_items ( old_items , new_items ) : <EOL> '''<STR_LIT>''' <EOL> changelist = Changelist ( ) <EOL> old_dict = dict ( ( item . path , item . store_path ) for item in old_items ) <EOL> new_dict = dict ( ( item . path , item . store_path ) for item in new_items ) <EOL> for new_item in new_items : <EOL> if new_item . path not in old_dict : <EOL> changelist . update ( new_item . path , Changelist . Status . ADDED ) <EOL> for old_item in old_items : <EOL> if old_item . path not in new_dict : <EOL> changelist . update ( old_item . path , Changelist . Status . REMOVED ) <EOL> elif old_item . store_path != new_dict [ old_item . path ] : <EOL> changelist . update ( old_item . path , Changelist . Status . MODIFIED ) <EOL> return changelist <EOL> @ log_calls <EOL> def changelist_for_fingerprints ( old_fingerprints , new_fingerprints , fill_md5s = True , lazy_md5s = False , root_path = None , expect_exact = True , mod_all = False ) : <EOL> '''<STR_LIT>''' <EOL> changelist = Changelist ( ) <EOL> if lazy_md5s and not root_path : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> if fill_md5s : <EOL> old_fingerprints . fill_md5s ( new_fingerprints ) <EOL> new_fingerprints . fill_md5s ( old_fingerprints ) <EOL> old_dict = old_fingerprints . fingerprints <EOL> new_dict = new_fingerprints . fingerprints <EOL> for new_path in new_dict . keys ( ) : <EOL> if new_path not in old_dict : <EOL> changelist . update ( new_path , Changelist . Status . ADDED ) <EOL> for old_path in old_dict . keys ( ) : <EOL> if old_path not in new_dict : <EOL> changelist . update ( old_path , Changelist . Status . REMOVED ) <EOL> else : <EOL> if mod_all : <EOL> changelist . update ( old_path , Changelist . Status . MODIFIED ) <EOL> else : <EOL> old_fp = old_dict [ old_path ] <EOL> new_fp = new_dict [ old_path ] <EOL> c = old_fp . compare ( new_fp ) <EOL> log . debug ( "<STR_LIT>" , old_path , c , old_fp , new_fp ) <EOL> if c == Fingerprint . UNCERTAIN and lazy_md5s : <EOL> if not old_fp . md5 : <EOL> old_fp . read_from_file ( join_path ( root_path , old_path ) , compute_md5 = True ) <EOL> if not new_fp . md5 : <EOL> new_fp . read_from_file ( join_path ( root_path , old_path ) , compute_md5 = True ) <EOL> c = old_fp . compare ( new_fp ) <EOL> log . debug ( "<STR_LIT>" , old_path , c ) <EOL> if c == Fingerprint . DIFFER : <EOL> changelist . update ( old_path , Changelist . Status . MODIFIED ) <EOL> elif c == Fingerprint . UNCERTAIN : <EOL> if lazy_md5s or expect_exact : <EOL> raise AssertionError ( "<STR_LIT>" % ( old_fp , new_fp , lazy_md5s , expect_exact ) ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , old_path ) <EOL> changelist . update ( old_path , Changelist . Status . MODIFIED ) <EOL> return changelist <EOL> class WorkingDir ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , cur_dir , config ) : <EOL> ( self . work_dir , _rel_path ) = self . work_dirs_for_path ( cur_dir , require = True ) [ <NUM_LIT:0> ] <EOL> self . config = config <EOL> wconfig = WorkingDir . _work_dir_config ( self . work_dir ) <EOL> if wconfig : <EOL> config . merge_in ( dict ( wconfig . items ( "<STR_LIT>" ) ) ) <EOL> if not config . root_uri : <EOL> raise InvalidOperation ( "<STR_LIT>" , cur_dir ) <EOL> if config . no_cache : <EOL> file_cache = None <EOL> else : <EOL> file_cache = FileCache ( self . _path_for_file_cache ( self . work_dir ) , cache_criterion = Repo . store_path_is_cachable ) <EOL> self . repo = Repo ( config . root_uri , self . config , file_cache = file_cache ) <EOL> self . _read_checkout_state ( ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % self . work_dir <EOL> def _get_relative_paths ( self , file_list , scope ) : <EOL> relative_paths = [ ] <EOL> prefix = join_path ( self . work_dir , scope ) + "<STR_LIT:/>" <EOL> for path in file_list : <EOL> if path . startswith ( prefix ) : <EOL> relative_paths . append ( path . replace ( prefix , '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> else : <EOL> relative_paths . append ( path ) <EOL> return relative_paths <EOL> def _walk_files ( self , scope , force_mode = Mode . PARTIAL , print_untracked = False ) : <EOL> '''<STR_LIT>''' <EOL> all_files = [ ] <EOL> scope_dir = join_path ( self . work_dir , scope ) <EOL> log . debug ( "<STR_LIT>" , scope_dir ) <EOL> not_partial = not ( force_mode == Mode . PARTIAL and self . get_scope_mode ( scope ) == Mode . PARTIAL ) <EOL> for root , dirnames , filenames in os . walk ( scope_dir ) : <EOL> for filename in filenames : <EOL> full_path = join_path ( root , filename ) <EOL> if not self . ignore_path ( full_path ) and ( not_partial or self . _is_tracking ( scope , strip_prefix ( self . work_dir + "<STR_LIT:/>" + scope + "<STR_LIT:/>" , full_path ) , print_untracked = print_untracked ) ) : <EOL> all_files . append ( full_path ) <EOL> return self . _get_relative_paths ( all_files , scope ) <EOL> def _get_absolute_path ( self , scope , path ) : <EOL> return join_path ( self . work_dir , scope , path ) <EOL> def _delete_file ( self , scope , filename , backup_suffix = None ) : <EOL> log . info ( "<STR_LIT>" , self . _get_absolute_path ( scope , filename ) ) <EOL> path = self . _get_absolute_path ( scope , filename ) <EOL> if backup_suffix : <EOL> move_to_backup ( path , backup_suffix = backup_suffix ) <EOL> else : <EOL> os . remove ( path ) <EOL> @ contextmanager <EOL> def _yield_fingerprint ( self , scope , path ) : <EOL> '''<STR_LIT>''' <EOL> local_path = join_path ( scope , path ) <EOL> fp = self . checkout_state . fingerprints . lookup_fingerprint ( local_path ) <EOL> yield ( fp , local_path ) <EOL> if fp is not None : <EOL> self . checkout_state . fingerprints . update_fingerprint ( local_path , fp ) <EOL> def _read_checkout_state ( self ) : <EOL> legacy_path = join_path ( self . work_dir , "<STR_LIT>" , "<STR_LIT>" ) <EOL> if os . path . exists ( legacy_path ) : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> checkout_state = CheckoutState ( self . work_dir ) <EOL> filename = self . _path_for_checkout_state ( self . work_dir ) <EOL> if os . path . exists ( filename ) : <EOL> log . debug ( "<STR_LIT>" , filename ) <EOL> checkout_state . from_str ( read_string_from_file ( filename ) ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" ) <EOL> self . checkout_state = checkout_state <EOL> def _write_checkout_state ( self ) : <EOL> filename = self . _path_for_checkout_state ( self . work_dir ) <EOL> log . debug ( "<STR_LIT>" , self . checkout_state , filename ) <EOL> write_string_to_file ( filename , self . checkout_state . to_str ( ) ) <EOL> def _is_tracking ( self , scope , path , print_untracked = False ) : <EOL> '''<STR_LIT>''' <EOL> assert self . checkout_state <EOL> tracking = False <EOL> with self . _yield_fingerprint ( scope , path ) as ( fp , local_path ) : <EOL> if fp is not None : <EOL> tracking = fp . state in FileStateTrackedList <EOL> if not tracking and print_untracked : <EOL> log . info ( "<STR_LIT>" % local_path ) <EOL> return tracking <EOL> def checkouts ( self , expect_nonempty = True ) : <EOL> '''<STR_LIT>''' <EOL> checkouts = self . checkout_state . checkouts . as_list ( ) <EOL> if expect_nonempty and len ( checkouts ) < <NUM_LIT:1> : <EOL> raise InvalidOperation ( "<STR_LIT>" , suppressable = True ) <EOL> return checkouts <EOL> def get_checkout_rev ( self , scope ) : <EOL> return self . checkout_state . checkouts . rev ( scope ) <EOL> def get_scope_mode ( self , scope ) : <EOL> return self . checkout_state . scope_mode ( scope ) <EOL> def update_scope_mode ( self , scope , mode ) : <EOL> self . checkout_state . scope_mode ( scope , mode ) <EOL> @ log_calls <EOL> def checkout ( self , scope , srev = "<STR_LIT>" , force = False , mode = Mode . AUTO ) : <EOL> '''<STR_LIT>''' <EOL> self . repo . resolve_rev ( scope , srev ) <EOL> self . _ensure_dot_dir_present ( self . work_dir ) <EOL> local_scope_path = join_path ( self . work_dir , scope ) <EOL> make_all_dirs ( local_scope_path ) <EOL> if mode is None : <EOL> mode = Mode . AUTO <EOL> mf = self . update ( scope , srev = srev , force = force , is_checkout = True , mode = mode ) <EOL> return mf <EOL> def _fingerprint_status ( self , scope , compute_md5 = False , force_mode = Mode . PARTIAL , print_untracked = False ) : <EOL> '''<STR_LIT>''' <EOL> if not self . checkout_state . has_scope ( scope ) : <EOL> raise InvalidArgument ( "<STR_LIT>" % scope ) <EOL> old_fingerprints = self . checkout_state . for_subdir ( scope ) <EOL> file_list = self . _walk_files ( scope , force_mode = force_mode , print_untracked = print_untracked ) <EOL> local_scope_path = join_path ( self . work_dir , scope ) <EOL> new_fingerprints = FingerprintList . of_files ( local_scope_path , file_list , compute_md5 = compute_md5 ) <EOL> return ( old_fingerprints , new_fingerprints ) <EOL> @ log_calls <EOL> def status ( self , scope , mod_all = False , force_mode = Mode . PARTIAL , full = False ) : <EOL> '''<STR_LIT>''' <EOL> if full : <EOL> log . info ( "<STR_LIT>" , "<STR_LIT:status>" , scope , self . get_checkout_rev ( scope ) , self . get_scope_mode ( scope ) ) <EOL> local_scope_dir = join_path ( self . work_dir , scope ) <EOL> ( old_fingerprints , new_fingerprints ) = self . _fingerprint_status ( scope , compute_md5 = False , force_mode = force_mode , print_untracked = full ) <EOL> changelist = changelist_for_fingerprints ( old_fingerprints , new_fingerprints , lazy_md5s = True , root_path = local_scope_dir , mod_all = mod_all ) <EOL> return changelist <EOL> def _update_local_files ( self , scope , items , changelist , delete_all = False , delete_items = False , backup_suffix = None ) : <EOL> '''<STR_LIT>''' <EOL> items_to_fetch = [ ] <EOL> for item in items : <EOL> if item . path in changelist . mod_paths or item . path in changelist . add_paths : <EOL> items_to_fetch . append ( item ) <EOL> local_scope_path = join_path ( self . work_dir , scope ) <EOL> new_fingerprints = self . repo . fetch_items ( scope , items_to_fetch , local_scope_path , clobber = True , backup_suffix = backup_suffix ) <EOL> if delete_all : <EOL> for path in sorted ( list ( changelist . rm_paths ) ) : <EOL> self . _delete_file ( scope , path , backup_suffix = backup_suffix ) <EOL> elif delete_items : <EOL> for item in items : <EOL> if item . path in changelist . rm_paths : <EOL> self . _delete_file ( scope , item . path , backup_suffix = backup_suffix ) <EOL> return new_fingerprints <EOL> @ log_calls <EOL> def update ( self , scope , srev = "<STR_LIT>" , force = False , is_checkout = False , mode = None ) : <EOL> '''<STR_LIT>''' <EOL> partial_mode = mode == Mode . PARTIAL <EOL> is_partial_checkout = is_checkout and partial_mode <EOL> cur_rev = self . get_checkout_rev ( scope ) <EOL> if is_checkout : <EOL> if cur_rev and not force : <EOL> raise InvalidOperation ( "<STR_LIT>" % scope ) <EOL> old_mf = Manifest ( ) <EOL> else : <EOL> if mode is not None : <EOL> raise InvalidOperation ( "<STR_LIT>" % mode ) <EOL> old_mf = self . repo . get_manifest ( scope , cur_rev ) if cur_rev else Manifest ( ) <EOL> new_mf = self . repo . get_manifest ( scope , srev ) <EOL> changelist = changelist_for_items ( old_mf . items , new_mf . items ) if not is_partial_checkout else Changelist ( ) <EOL> log . debug ( "<STR_LIT>" , old_mf , new_mf , changelist ) <EOL> if partial_mode or self . get_scope_mode ( scope ) == Mode . PARTIAL : <EOL> for path in changelist . all_paths ( ) : <EOL> with self . _yield_fingerprint ( scope , path ) as ( fp , local_path ) : <EOL> if fp is None or fp . state not in FileStateTrackedList : <EOL> if fp is not None : <EOL> fp . invalidate ( ) <EOL> changelist . discard ( path ) <EOL> status_changelist = self . status ( scope ) if not is_checkout else Changelist ( ) <EOL> overlap_changelist = status_changelist . intersect ( changelist ) <EOL> log . debug ( "<STR_LIT>" , changelist . to_summary ( ) ) <EOL> log . debug ( "<STR_LIT>" , status_changelist . to_summary ( ) ) <EOL> if not overlap_changelist . is_empty ( ) : <EOL> log . error ( "<STR_LIT>" , scope ) <EOL> log . stream . write ( overlap_changelist . to_summary ( ) ) <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> log . info ( "<STR_LIT>" , <EOL> "<STR_LIT>" if is_checkout else "<STR_LIT>" , scope , new_mf . rev , len ( new_mf . items ) ) <EOL> conflict_str = "<STR_LIT>" if status_changelist . is_empty ( ) else "<STR_LIT>" % status_changelist . brief_summary ( ) <EOL> if not changelist . is_empty ( ) : <EOL> log . info ( "<STR_LIT>" , changelist . brief_summary ( ) , conflict_str ) <EOL> new_fingerprints = self . _update_local_files ( scope , new_mf . items , changelist , delete_all = True ) <EOL> self . checkout_state . update_from_changelist ( scope , changelist , new_fingerprints ) <EOL> self . checkout_state . update_checkout_rev ( scope , new_mf . rev ) <EOL> if is_partial_checkout : <EOL> self . update_scope_mode ( scope , Mode . PARTIAL ) <EOL> self . _write_checkout_state ( ) <EOL> return new_mf <EOL> @ log_calls <EOL> def track ( self , scope , track_path_list = [ ] , mode = None , no_download = False , recursive = False ) : <EOL> '''<STR_LIT>''' <EOL> if mode is None : <EOL> partial_mode = self . get_scope_mode ( scope ) == Mode . PARTIAL <EOL> if not partial_mode and track_path_list : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( scope , Mode . PARTIAL ) ) <EOL> else : <EOL> partial_mode = mode == Mode . PARTIAL and ( len ( track_path_list ) > <NUM_LIT:0> or self . get_scope_mode ( scope ) != Mode . PARTIAL ) <EOL> partial2auto_mode = self . get_scope_mode ( scope ) == Mode . PARTIAL and mode == Mode . AUTO and len ( track_path_list ) == <NUM_LIT:0> <EOL> if not ( partial_mode or partial2auto_mode ) : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( scope , Mode . AUTO , mode ) ) <EOL> cur_rev = self . get_checkout_rev ( scope ) <EOL> mf = self . repo . get_manifest ( scope , cur_rev ) <EOL> status_changelist = self . status ( scope , force_mode = Mode . AUTO ) <EOL> changelist = changelist_for_items ( [ ] , mf . items ) <EOL> log . debug ( "<STR_LIT>" , mf , changelist ) <EOL> track_changelist = Changelist ( ) <EOL> if partial_mode : <EOL> self . update_scope_mode ( scope , Mode . PARTIAL ) <EOL> log . info ( "<STR_LIT>" , "<STR_LIT>" , scope , mf . rev , Mode . PARTIAL ) <EOL> path_candidates = [ item . path for item in mf . items ] + list ( status_changelist . add_paths ) <EOL> for path in expand_wildcards ( track_path_list , path_candidates , recursive = recursive ) : <EOL> with self . _yield_fingerprint ( scope , path ) as ( fp , local_path ) : <EOL> if fp is not None and fp . state in FileStateTrackedList : <EOL> log . warn ( "<STR_LIT>" , scope , path , fp . state ) <EOL> changelist . discard ( path ) <EOL> else : <EOL> track_changelist . update ( path , Changelist . Status . ADDED ) <EOL> changelist = changelist . intersect ( track_changelist ) <EOL> if partial2auto_mode : <EOL> self . update_scope_mode ( scope , Mode . AUTO ) <EOL> log . info ( "<STR_LIT>" , "<STR_LIT>" , scope , mf . rev , Mode . AUTO ) <EOL> mod_changelist = status_changelist . intersect ( changelist ) <EOL> add_changelist = track_changelist . difference ( changelist ) <EOL> retrack_changelist = changelist . difference ( mod_changelist ) <EOL> retrack_changelist_paths = retrack_changelist . all_paths ( ) <EOL> for path in retrack_changelist_paths : <EOL> with self . _yield_fingerprint ( scope , path ) as ( fp , local_path ) : <EOL> if fp is not None and fp . state in FileStateTrackedList : <EOL> self . checkout_state . _check_file_exists ( local_path ) <EOL> retrack_changelist . discard ( path ) <EOL> log . debug ( "<STR_LIT>" , changelist . to_summary ( ) ) <EOL> log . debug ( "<STR_LIT>" , status_changelist . to_summary ( ) ) <EOL> log . debug ( "<STR_LIT>" , mod_changelist . to_summary ( ) ) <EOL> log . debug ( "<STR_LIT>" , add_changelist . to_summary ( ) ) <EOL> log . debug ( "<STR_LIT>" , retrack_changelist . to_summary ( ) ) <EOL> if mod_changelist . is_empty ( ) : <EOL> log . info ( "<STR_LIT>" ) <EOL> else : <EOL> log . info ( "<STR_LIT>" , mod_changelist . brief_summary ( ) ) <EOL> for path in mod_changelist . all_paths ( ) : <EOL> with self . _yield_fingerprint ( scope , path ) as ( fp , local_path ) : <EOL> if fp is None : <EOL> assert False <EOL> fp . state = FileState . Tracked <EOL> fp . md5 = INVALID_MD5 <EOL> if not add_changelist . is_empty ( ) : <EOL> add_changelist_paths = add_changelist . all_paths ( ) <EOL> for path in add_changelist_paths : <EOL> local_path = join_path ( scope , path ) <EOL> self . checkout_state . _check_file_exists ( local_path ) <EOL> log . info ( "<STR_LIT>" , add_changelist . brief_summary ( ) ) <EOL> local_scope_path = join_path ( self . work_dir , scope ) <EOL> new_fingerprints = FingerprintList . of_files ( local_scope_path , add_changelist_paths , compute_md5 = True , state = FileState . ToBeTracked ) <EOL> self . checkout_state . update_from_changelist ( scope , add_changelist , new_fingerprints ) <EOL> if retrack_changelist . is_empty ( ) : <EOL> if no_download : <EOL> log . warn ( "<STR_LIT>" ) <EOL> else : <EOL> log . info ( "<STR_LIT>" , retrack_changelist . brief_summary ( ) ) <EOL> if no_download : <EOL> for path in retrack_changelist_paths : <EOL> local_path = join_path ( scope , path ) <EOL> fp = Fingerprint ( md5 = INVALID_MD5 , mtime = INVALID_MTIME , size = INVALID_SIZE , state = FileState . Tracked ) <EOL> self . checkout_state . fingerprints . update_fingerprint ( local_path , fp ) <EOL> else : <EOL> new_fingerprints = self . _update_local_files ( scope , mf . items , retrack_changelist , delete_all = True ) <EOL> self . checkout_state . update_from_changelist ( scope , retrack_changelist , new_fingerprints ) <EOL> self . _write_checkout_state ( ) <EOL> return mf <EOL> @ log_calls <EOL> def untrack ( self , scope , untrack_path_list , mode , recursive = False ) : <EOL> if Mode . PARTIAL not in [ self . get_scope_mode ( scope ) , mode ] : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( scope , Mode . PARTIAL ) ) <EOL> self . update_scope_mode ( scope , Mode . PARTIAL ) <EOL> path_candidates = [ ] <EOL> for local_path in self . checkout_state . fingerprints . fingerprints . iterkeys ( ) : <EOL> path = strip_prefix ( scope + '<STR_LIT:/>' , local_path ) <EOL> if path is not None : <EOL> path_candidates . append ( path ) <EOL> for path in expand_wildcards ( untrack_path_list , path_candidates , recursive = recursive ) : <EOL> with self . _yield_fingerprint ( scope , path ) as ( fp , local_path ) : <EOL> if fp is None or fp . state == FileState . Untracked : <EOL> log . warn ( "<STR_LIT>" , scope , path , FileState . Untracked ) <EOL> else : <EOL> fp . state = FileState . Untracked <EOL> self . _write_checkout_state ( ) <EOL> @ log_calls <EOL> def diff ( self , scope , opt_rev = None , path_list = None ) : <EOL> '''<STR_LIT>''' <EOL> changelist = self . status ( scope ) <EOL> cur_rev = self . get_checkout_rev ( scope ) <EOL> base_rev = None <EOL> trgt_rev = None <EOL> base_msg = "<STR_LIT>" <EOL> trgt_msg = "<STR_LIT>" <EOL> if opt_rev : <EOL> ( base_rev , trgt_rev ) = parse_rev_range ( opt_rev ) <EOL> if not base_rev : <EOL> raise InvalidArgument ( "<STR_LIT>" % opt_rev ) <EOL> base_msg = "<STR_LIT>" % base_rev <EOL> if trgt_rev : <EOL> trgt_msg = "<STR_LIT>" % trgt_rev <EOL> else : <EOL> if changelist . is_empty ( ) : <EOL> trgt_msg = "<STR_LIT>" <EOL> trgt_rev = cur_rev <EOL> else : <EOL> trgt_msg = "<STR_LIT>" <EOL> base_mf = self . repo . get_manifest ( scope , base_rev ) <EOL> trgt_mf = self . repo . get_manifest ( scope , ( trgt_rev if trgt_rev else cur_rev ) ) <EOL> if trgt_rev : <EOL> changelist = changelist_for_items ( base_mf . items , trgt_mf . items ) <EOL> else : <EOL> changelist = changelist_for_items ( base_mf . items , trgt_mf . items ) . union ( changelist ) <EOL> elif not changelist . is_empty ( ) : <EOL> base_msg = "<STR_LIT>" <EOL> trgt_msg = "<STR_LIT>" <EOL> base_rev = cur_rev <EOL> if base_rev : <EOL> if path_list : <EOL> tmp_changelist = Changelist ( ) <EOL> for path in path_list : <EOL> tmp_changelist . update ( path , Changelist . Status . MODIFIED ) <EOL> changelist = changelist . intersect ( tmp_changelist ) <EOL> if not changelist . is_empty ( ) : <EOL> sys . stdout . write ( "<STR_LIT>" % ( base_msg , trgt_msg ) ) <EOL> assert scope <EOL> sys . stdout . write ( "<STR_LIT>" % scope ) <EOL> if path_list : <EOL> sys . stdout . write ( "<STR_LIT>" % path_list ) <EOL> def _do_print_diff ( path_a , path_b ) : <EOL> @ contextmanager <EOL> def _get_real_path_and_label ( path , rev = None ) : <EOL> if not path : <EOL> yield ( EMPTY_FILEPATH , EMPTY_FILEPATH ) <EOL> else : <EOL> label = join_path ( scope , path ) <EOL> if rev : <EOL> items = self . repo . list ( scope , path , rev , recursive = False ) <EOL> assert len ( items ) == <NUM_LIT:1> <EOL> store_path = join_path ( self . repo . store_path_dir ( scope ) , items [ <NUM_LIT:0> ] . store_path ) <EOL> ( cache_path , fp ) = self . repo . store . cache ( store_path ) <EOL> if not cache_path : <EOL> raise InvalidOperation ( "<STR_LIT>" % store_path ) <EOL> with decompressed_local_file ( cache_path , items [ <NUM_LIT:0> ] . scheme ) as decompressed_path : <EOL> yield ( decompressed_path , label ) <EOL> else : <EOL> full_path = join_path ( self . work_dir , scope , path ) <EOL> yield ( full_path , label ) <EOL> with _get_real_path_and_label ( path_a , base_rev ) as ( base_path , base_label ) : <EOL> with _get_real_path_and_label ( path_b , trgt_rev ) as ( trgt_path , trgt_label ) : <EOL> print_diff ( base_path , trgt_path , base_label , trgt_label ) <EOL> for path in changelist . add_paths : <EOL> _do_print_diff ( None , path ) <EOL> for path in changelist . rm_paths : <EOL> _do_print_diff ( path , None ) <EOL> for path in changelist . mod_paths : <EOL> _do_print_diff ( path , path ) <EOL> @ log_calls <EOL> def revert ( self , scope , path_list = None , backup_suffix = BACKUP_SUFFIX ) : <EOL> '''<STR_LIT>''' <EOL> full_revert = not path_list <EOL> rev = self . get_checkout_rev ( scope ) <EOL> mf = self . repo . get_manifest ( scope , rev ) <EOL> local_scope_dir = join_path ( self . work_dir , scope ) <EOL> items = mf . get_items ( path_list ) if path_list else mf . items <EOL> if None in items : <EOL> raise InvalidArgument ( "<STR_LIT>" % ( rev , scope , path_list [ items . index ( None ) ] ) ) <EOL> ( old_fingerprints , new_fingerprints ) = self . _fingerprint_status ( scope , compute_md5 = False ) <EOL> changelist = changelist_for_fingerprints ( new_fingerprints , old_fingerprints , lazy_md5s = True , root_path = local_scope_dir ) <EOL> new_fingerprints = self . _update_local_files ( scope , items , changelist , delete_items = True , delete_all = full_revert , backup_suffix = BACKUP_SUFFIX ) <EOL> self . checkout_state . update_fingerprints ( scope , new_fingerprints ) <EOL> self . _write_checkout_state ( ) <EOL> @ log_calls <EOL> def commit ( self , scope , user = UNKNOWN_USER , message = EMPTY_MESSAGE , time = None , empty_ok = False , mod_all = False ) : <EOL> '''<STR_LIT>''' <EOL> repo_tip = self . repo . get_tip ( scope ) <EOL> prev_rev = self . repo . resolve_rev ( scope , self . get_checkout_rev ( scope ) ) <EOL> if prev_rev != repo_tip : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( repo_tip , prev_rev , scope ) ) <EOL> local_scope_dir = join_path ( self . work_dir , scope ) <EOL> ( old_fingerprints , new_fingerprints ) = self . _fingerprint_status ( scope , compute_md5 = False ) <EOL> changelist = changelist_for_fingerprints ( old_fingerprints , new_fingerprints , lazy_md5s = True , root_path = local_scope_dir , mod_all = mod_all ) <EOL> new_fingerprints . compute_md5s ( local_scope_dir ) <EOL> if changelist . is_empty ( ) : <EOL> if empty_ok : <EOL> return <EOL> else : <EOL> raise InvalidOperation ( "<STR_LIT>" % scope , suppressable = True ) <EOL> for local_path , fp in self . checkout_state . fingerprints . fingerprints . iteritems ( ) : <EOL> if fp . state == FileState . ToBeTracked : <EOL> self . checkout_state . _check_file_exists ( local_path ) <EOL> log . info ( "<STR_LIT>" , changelist . brief_summary ( ) ) <EOL> log . stream . write ( changelist . to_summary ( prefix = scope ) ) <EOL> rev = self . repo . commit ( scope , local_scope_dir , prev_rev , changelist , user = user , message = message , time = time ) <EOL> self . checkout_state . update_from_changelist ( scope , changelist , new_fingerprints ) <EOL> self . checkout_state . update_checkout_rev ( scope , rev ) <EOL> self . _write_checkout_state ( ) <EOL> @ staticmethod <EOL> def ignore_path ( full_path ) : <EOL> '''<STR_LIT>''' <EOL> dirname = os . path . dirname ( full_path ) <EOL> filename = os . path . basename ( full_path ) <EOL> def ignore_path_elem ( c ) : <EOL> return c . startswith ( TMP_DIR_PREFIX ) <EOL> ignore_dir = any ( ignore_path_elem ( c ) for c in dirname . split ( "<STR_LIT:/>" ) ) <EOL> return ignore_dir or filename . startswith ( "<STR_LIT:.>" ) or filename . startswith ( "<STR_LIT>" ) or filename . endswith ( "<STR_LIT>" ) or filename . endswith ( "<STR_LIT>" ) or filename . endswith ( "<STR_LIT>" ) or filename . endswith ( BACKUP_SUFFIX ) <EOL> @ staticmethod <EOL> def _ensure_dot_dir_present ( work_dir ) : <EOL> dot_dir = WorkingDir . _path_for_dot_dir ( work_dir ) <EOL> if not os . path . exists ( dot_dir ) : <EOL> log . debug ( "<STR_LIT>" , dot_dir ) <EOL> os . makedirs ( dot_dir ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , dot_dir ) <EOL> @ staticmethod <EOL> def initialize_work_dir ( path , root_uri ) : <EOL> '''<STR_LIT>''' <EOL> pairs = WorkingDir . work_dirs_for_path ( path ) <EOL> if len ( pairs ) > <NUM_LIT:1> : <EOL> raise InvalidOperation ( "<STR_LIT>" % "<STR_LIT:U+002CU+0020>" . join ( [ work_dir for ( work_dir , rel_path ) in pairs ] ) ) <EOL> elif len ( pairs ) == <NUM_LIT:1> : <EOL> ( work_dir , _rel_path ) = pairs [ <NUM_LIT:0> ] <EOL> if path != work_dir : <EOL> raise InvalidOperation ( "<STR_LIT>" % ( path , work_dir ) ) <EOL> make_all_dirs ( path ) <EOL> return WorkingDir . _update_work_dir_config ( path , { "<STR_LIT>" : root_uri } ) <EOL> @ staticmethod <EOL> def work_dirs_for_path ( path , require = False ) : <EOL> '''<STR_LIT>''' <EOL> out = [ ] <EOL> all_dot_dirs = [ ] <EOL> abs_path = os . path . abspath ( path ) <EOL> return_abs_paths = os . path . isabs ( path ) <EOL> def simplify_path ( p ) : <EOL> return p if return_abs_paths else os . path . relpath ( p ) <EOL> for ( parent , rel_path ) in parent_dirs ( abs_path ) : <EOL> dot_dir = WorkingDir . _path_for_dot_dir ( parent ) <EOL> if os . path . isdir ( dot_dir ) : <EOL> log . debug ( "<STR_LIT>" , parent , rel_path , path ) <EOL> out . append ( ( simplify_path ( parent ) , rel_path ) ) <EOL> all_dot_dirs . append ( dot_dir ) <EOL> if require and not out : <EOL> raise InvalidOperation ( "<STR_LIT>" % path ) <EOL> if len ( all_dot_dirs ) > <NUM_LIT:1> : <EOL> log . error ( "<STR_LIT>" , "<STR_LIT:U+002CU+0020>" . join ( all_dot_dirs ) ) <EOL> return out <EOL> @ staticmethod <EOL> def _work_dir_config ( work_dir ) : <EOL> '''<STR_LIT>''' <EOL> path = WorkingDir . _path_for_config ( work_dir ) <EOL> wconfig = None <EOL> if os . path . exists ( path ) : <EOL> wconfig = ConfigParser . ConfigParser ( ) <EOL> log . debug ( "<STR_LIT>" , path ) <EOL> wconfig . read ( path ) <EOL> else : <EOL> log . debug ( "<STR_LIT>" , path ) <EOL> return wconfig <EOL> @ log_calls <EOL> def enclosing_scope ( self , path ) : <EOL> scopes = set ( [ c . scope for c in self . checkouts ( ) ] ) <EOL> for ( parent_path , _rel_path ) in parent_dirs ( path ) : <EOL> if parent_path in scopes : <EOL> return parent_path <EOL> return None <EOL> @ staticmethod <EOL> def work_dir_context_for_path ( path , config ) : <EOL> '''<STR_LIT>''' <EOL> pairs = WorkingDir . work_dirs_for_path ( path , require = False ) <EOL> if len ( pairs ) > <NUM_LIT:0> : <EOL> ( work_dir , rel_path ) = pairs [ <NUM_LIT:0> ] <EOL> work = WorkingDir ( work_dir , config ) <EOL> scope = work . enclosing_scope ( rel_path ) <EOL> return ( work , rel_path , scope ) <EOL> else : <EOL> return ( None , None , None ) <EOL> @ staticmethod <EOL> def _update_work_dir_config ( work_dir , values ) : <EOL> '''<STR_LIT>''' <EOL> wconfig = WorkingDir . _work_dir_config ( work_dir ) <EOL> first_time = False <EOL> if not wconfig : <EOL> first_time = True <EOL> wconfig = ConfigParser . ConfigParser ( ) <EOL> if not wconfig . has_section ( "<STR_LIT>" ) : <EOL> wconfig . add_section ( "<STR_LIT>" ) <EOL> for ( k , v ) in values . items ( ) : <EOL> wconfig . set ( "<STR_LIT>" , k , v ) <EOL> WorkingDir . _ensure_dot_dir_present ( work_dir ) <EOL> config_path = WorkingDir . _path_for_config ( work_dir ) <EOL> log . debug ( "<STR_LIT>" , config_path ) <EOL> with open ( config_path , "<STR_LIT:w>" ) as handler : <EOL> wconfig . write ( handler ) <EOL> return first_time <EOL> @ staticmethod <EOL> def _path_for_dot_dir ( work_dir ) : <EOL> return join_path ( work_dir , "<STR_LIT>" ) <EOL> @ staticmethod <EOL> def _path_for_config ( work_dir ) : <EOL> return join_path ( work_dir , "<STR_LIT>" , "<STR_LIT>" ) <EOL> @ staticmethod <EOL> def _path_for_checkout_state ( work_dir ) : <EOL> return join_path ( work_dir , "<STR_LIT>" , "<STR_LIT>" ) <EOL> @ staticmethod <EOL> def _path_for_file_cache ( work_dir ) : <EOL> return join_path ( work_dir , "<STR_LIT>" , "<STR_LIT>" ) <EOL> _COMMAND_LIST_REPO = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:list>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> _COMMAND_LIST_WORK = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:id>" , "<STR_LIT>" , "<STR_LIT:status>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> _COMMAND_ABBREVS = { "<STR_LIT>" : "<STR_LIT>" , "<STR_LIT>" : "<STR_LIT:list>" } <EOL> def require_opt ( command , value , options , test_func = lambda x : x is not None ) : <EOL> '''<STR_LIT>''' <EOL> if not test_func ( getattr ( options , value ) ) : <EOL> fail ( "<STR_LIT>" % ( command , value ) ) <EOL> def require_arg ( command , position , description , command_args ) : <EOL> '''<STR_LIT>''' <EOL> if len ( command_args ) < position : <EOL> fail ( "<STR_LIT>" % ( command , position , description ) ) <EOL> return command_args [ position - <NUM_LIT:1> ] <EOL> def run_command ( command , command_args , options ) : <EOL> '''<STR_LIT>''' <EOL> cwd = os . getcwd ( ) <EOL> if command in _COMMAND_ABBREVS : <EOL> command = _COMMAND_ABBREVS [ command ] <EOL> config = Config ( options = options ) <EOL> if options . scope : <EOL> options . scope = options . scope . rstrip ( '<STR_LIT:/>' ) <EOL> def populate_options_from_cwd ( cwd , options ) : <EOL> ( cwd_work , cwd_rel_path , cwd_scope ) = WorkingDir . work_dir_context_for_path ( cwd , config ) <EOL> cwd_work_dir = cwd_work . work_dir if cwd_work else None <EOL> cwd_root_uri = cwd_work . config . root_uri if cwd_work else None <EOL> if options . root_uri and cwd_root_uri and cwd_root_uri != options . root_uri : <EOL> fail ( "<STR_LIT>" % ( cwd_root_uri , options . root_uri ) ) <EOL> if not options . root_uri and cwd_root_uri : <EOL> options . root_uri = cwd_root_uri <EOL> if not options . scope and cwd_scope : <EOL> options . scope = cwd_scope <EOL> if not options . work_dir and cwd_work_dir : <EOL> options . work_dir = cwd_work_dir <EOL> if command in _COMMAND_LIST_REPO : <EOL> if command == "<STR_LIT>" : <EOL> options . root_uri = require_arg ( command , <NUM_LIT:1> , "<STR_LIT>" , command_args ) <EOL> repo = Repo ( options . root_uri , config , check_valid = False ) <EOL> repo . init_repo ( ) <EOL> else : <EOL> populate_options_from_cwd ( cwd , options ) <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> repo = Repo ( options . root_uri , config , check_valid = True ) <EOL> if command == "<STR_LIT>" : <EOL> scope = require_arg ( command , <NUM_LIT:1> , "<STR_LIT>" , command_args ) . rstrip ( '<STR_LIT:/>' ) <EOL> require_opt ( command , "<STR_LIT:user>" , options ) <EOL> commit_time = parse_datetime ( options . date ) if options . date else None <EOL> repo . create_scope ( scope , user = options . user , message = options . message , time = commit_time ) <EOL> if command == "<STR_LIT>" : <EOL> scopes = repo . get_scopes ( ) <EOL> for scope in scopes : <EOL> print "<STR_LIT:%s>" % scope <EOL> elif command == "<STR_LIT>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> ( start , end ) = determine_rev_range ( options , config ) <EOL> time_filter = parse_datespec ( options . date , assume_utc = options . assume_utc ) if options . date else None <EOL> filter = ( lambda mf : time_filter ( mf . time ) ) if time_filter else None <EOL> repo . log ( options . scope , sys . stdout , srev_start = start , srev_end = end , filter = filter ) <EOL> elif command == "<STR_LIT>" : <EOL> work = None <EOL> if options . work_dir : <EOL> log . debug ( "<STR_LIT>" , options . work_dir ) <EOL> work = WorkingDir ( options . work_dir , config ) <EOL> tag_name = require_arg ( command , <NUM_LIT:1> , "<STR_LIT>" , command_args ) <EOL> if options . all : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> if options . work : <EOL> if options . rev : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> if not work : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> checkouts = work . checkouts ( ) <EOL> for checkout in checkouts : <EOL> log . info ( "<STR_LIT>" , checkout . scope , tag_name , checkout . rev ) <EOL> repo . tag ( checkout . scope , tag_name , checkout . rev ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> if not options . rev and work : <EOL> rev = work . get_checkout_rev ( options . scope ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> rev = options . rev <EOL> log . info ( "<STR_LIT>" , options . scope , tag_name , rev ) <EOL> repo . tag ( options . scope , tag_name , rev ) <EOL> elif command == "<STR_LIT>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> ( start , end ) = parse_rev_range ( options . rev ) if options . rev else ( "<STR_LIT>" , None ) <EOL> time_filter = parse_datespec ( options . date , assume_utc = options . assume_utc ) if options . date else None <EOL> filter = ( lambda mf : time_filter ( mf . time ) ) if time_filter else None <EOL> tags = repo . get_tags ( options . scope , srev_start = start , srev_end = end , filter = filter ) <EOL> for ( name , rev ) in tags : <EOL> print "<STR_LIT>" % ( name , rev ) <EOL> elif command == "<STR_LIT:list>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> rev = options . rev if options . rev else "<STR_LIT>" <EOL> path = command_args [ <NUM_LIT:0> ] if len ( command_args ) > <NUM_LIT:0> else "<STR_LIT>" <EOL> items = repo . list ( options . scope , path , rev , recursive = options . recursive ) <EOL> prefix = None if options . short_paths else options . scope <EOL> for item in items : <EOL> print item . display_path ( prefix = prefix ) <EOL> elif command == "<STR_LIT>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> rev = options . rev if options . rev else "<STR_LIT>" <EOL> path = command_args [ <NUM_LIT:0> ] if len ( command_args ) > <NUM_LIT:0> else "<STR_LIT>" <EOL> items = repo . locate ( options . scope , path , rev , recursive = options . recursive ) <EOL> for item in items : <EOL> print "<STR_LIT>" % ( item [ <NUM_LIT:0> ] , item [ <NUM_LIT:1> ] ) <EOL> elif command == "<STR_LIT>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> rev = options . rev if options . rev else "<STR_LIT>" <EOL> src_path = require_arg ( command , <NUM_LIT:1> , "<STR_LIT>" , command_args ) <EOL> target_path = require_arg ( command , <NUM_LIT:2> , "<STR_LIT>" , command_args ) <EOL> repo . copy ( options . scope , src_path , rev , target_path , clobber = options . force , recursive = options . recursive ) <EOL> elif command == "<STR_LIT>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> rev = options . rev if options . rev else "<STR_LIT>" <EOL> mf = repo . get_manifest ( options . scope , rev ) <EOL> print mf . to_str ( ) <EOL> elif command in _COMMAND_LIST_WORK : <EOL> if options . mode is not None : <EOL> options . mode = options . mode . lower ( ) <EOL> if options . mode not in Mode . values ( ) : <EOL> raise InvalidArgument ( "<STR_LIT>" % Mode . values ( ) ) <EOL> if command == "<STR_LIT>" : <EOL> rev = options . rev if options . rev else "<STR_LIT>" <EOL> options . root_uri = require_arg ( command , <NUM_LIT:1> , "<STR_LIT>" , command_args ) <EOL> options . work_dir = command_args [ <NUM_LIT:1> ] if len ( command_args ) > <NUM_LIT:1> else cwd <EOL> if not options . all : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> Repo ( options . root_uri , config , check_valid = True ) . resolve_rev ( options . scope , rev ) <EOL> first_time = WorkingDir . initialize_work_dir ( options . work_dir , options . root_uri ) <EOL> if first_time : <EOL> log . info ( "<STR_LIT>" , options . work_dir ) <EOL> work = WorkingDir ( options . work_dir , config ) <EOL> if options . all : <EOL> if rev != "<STR_LIT>" : <EOL> raise InvalidArgument ( "<STR_LIT>" ) <EOL> scopes = work . repo . get_scopes ( ) <EOL> if len ( scopes ) < <NUM_LIT:1> : <EOL> raise InvalidOperation ( "<STR_LIT>" , suppressable = True ) <EOL> log . info ( "<STR_LIT>" , len ( scopes ) ) <EOL> for scope in scopes : <EOL> work . checkout ( scope , rev , force = options . force , mode = options . mode ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> work . checkout ( options . scope , rev , force = options . force , mode = options . mode ) <EOL> else : <EOL> populate_options_from_cwd ( cwd , options ) <EOL> work = WorkingDir ( options . work_dir , config ) <EOL> if command == "<STR_LIT:id>" : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> rev = work . get_checkout_rev ( options . scope ) <EOL> if not rev : <EOL> raise InvalidOperation ( "<STR_LIT>" % options . scope ) <EOL> print "<STR_LIT:%s>" % rev <EOL> elif command == "<STR_LIT>" : <EOL> checkouts = work . checkouts ( ) <EOL> for checkout in checkouts : <EOL> print "<STR_LIT>" % ( checkout . scope , checkout . rev ) <EOL> elif command == "<STR_LIT:status>" : <EOL> if options . all : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> elif options . work : <EOL> checkouts = work . checkouts ( ) <EOL> for checkout in checkouts : <EOL> prefix = None if options . short_paths else checkout . scope <EOL> sys . stdout . write ( work . status ( checkout . scope , mod_all = options . mod_all , full = options . full ) . to_summary ( prefix = prefix ) ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> log . debug ( "<STR_LIT>" , options . scope ) <EOL> prefix = None if options . short_paths else options . scope <EOL> sys . stdout . write ( work . status ( options . scope , mod_all = options . mod_all , full = options . full ) . to_summary ( prefix = prefix ) ) <EOL> elif command == "<STR_LIT>" : <EOL> if options . mode is not None : <EOL> raise InvalidOperation ( "<STR_LIT>" % command ) <EOL> require_opt ( command , "<STR_LIT:user>" , options ) <EOL> require_opt ( command , "<STR_LIT:message>" , options ) <EOL> commit_time = parse_datetime ( options . date , assume_utc = options . assume_utc ) if options . date else None <EOL> if options . all : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> elif options . work : <EOL> checkouts = work . checkouts ( ) <EOL> for checkout in checkouts : <EOL> log . info ( "<STR_LIT>" , checkout . scope ) <EOL> work . commit ( checkout . scope , user = options . user , message = options . message , time = commit_time , empty_ok = True , mod_all = options . mod_all ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> work . commit ( options . scope , user = options . user , message = options . message , time = commit_time , mod_all = options . mod_all ) <EOL> elif command == "<STR_LIT>" : <EOL> rev = options . rev if options . rev else "<STR_LIT>" <EOL> if not options . all and not options . work : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> work . update ( options . scope , rev , force = options . force ) <EOL> else : <EOL> if rev != "<STR_LIT>" : <EOL> raise InvalidArgument ( "<STR_LIT>" ) <EOL> checkouts = work . checkouts ( ) <EOL> scopes_to_checkout = None <EOL> if options . work : <EOL> log . info ( "<STR_LIT>" , len ( checkouts ) ) <EOL> else : <EOL> scopes_checked_out = set ( checkout . scope for checkout in checkouts ) <EOL> scopes_to_checkout = [ scope for scope in work . repo . get_scopes ( ) if scope not in scopes_checked_out ] <EOL> log . info ( "<STR_LIT>" , len ( scopes_checked_out ) , len ( scopes_to_checkout ) ) <EOL> for checkout in checkouts : <EOL> work . update ( checkout . scope , rev , force = options . force ) <EOL> if scopes_to_checkout : <EOL> for scope in scopes_to_checkout : <EOL> work . checkout ( scope , rev , force = options . force , mode = options . mode ) <EOL> elif command == "<STR_LIT>" : <EOL> if options . mode is not None : <EOL> raise InvalidOperation ( "<STR_LIT>" % command ) <EOL> if options . all : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> elif options . rev : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> elif options . work : <EOL> checkouts = work . checkouts ( ) <EOL> if len ( command_args ) > <NUM_LIT:0> : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> for checkout in checkouts : <EOL> log . info ( "<STR_LIT>" , checkout . scope ) <EOL> work . revert ( checkout . scope ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> work . revert ( options . scope , path_list = command_args ) <EOL> elif command in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> if options . all : <EOL> raise InvalidOperation ( "<STR_LIT>" % command ) <EOL> elif options . rev : <EOL> raise NotImplementedError ( "<STR_LIT>" % command ) <EOL> elif ( options . mode is None and not command_args ) or ( options . mode == Mode . AUTO and ( command == "<STR_LIT>" or ( command == "<STR_LIT>" and len ( command_args ) > <NUM_LIT:0> ) ) ) : <EOL> raise InvalidOperation ( "<STR_LIT>" % "<STR_LIT:U+0020>" . join ( [ command , "<STR_LIT>" + options . mode if options . mode else "<STR_LIT>" ] + command_args ) ) <EOL> elif options . force : <EOL> raise InvalidOperation ( "<STR_LIT>" % command ) <EOL> elif options . work : <EOL> checkouts = work . checkouts ( ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> checkouts = [ Checkout ( options . scope , work . get_checkout_rev ( options . scope ) , None ) ] <EOL> if command == "<STR_LIT>" : <EOL> if len ( command_args ) > <NUM_LIT:0> : <EOL> for checkout in checkouts : <EOL> work . track ( checkout . scope , track_path_list = command_args , mode = options . mode , no_download = options . no_download , recursive = options . recursive ) <EOL> else : <EOL> for checkout in checkouts : <EOL> work . track ( checkout . scope , mode = options . mode , no_download = options . no_download , recursive = options . recursive ) <EOL> elif command == "<STR_LIT>" : <EOL> for checkout in checkouts : <EOL> work . untrack ( checkout . scope , command_args , options . mode , recursive = options . recursive ) <EOL> else : <EOL> assert False <EOL> elif command == "<STR_LIT>" : <EOL> if options . all or options . date or options . mod_all or options . no_cache or options . work : <EOL> raise InvalidOperation ( "<STR_LIT>" ) <EOL> elif options . short_paths : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> else : <EOL> require_opt ( command , "<STR_LIT>" , options ) <EOL> work . diff ( options . scope , options . rev , path_list = command_args ) <EOL> else : <EOL> assert False <EOL> def main ( ) : <EOL> general_setup ( ) <EOL> if not sys . argv [ <NUM_LIT:0> ] : sys . argv [ <NUM_LIT:0> ] = "<STR_LIT>" <EOL> version_str = "<STR_LIT>" % ( ZINC_VERSION , REPO_VERSION , BOTO_VERSION ) <EOL> commands_str = "<STR_LIT:|>" . join ( sorted ( list ( _COMMAND_LIST_REPO + _COMMAND_LIST_WORK ) , key = lambda s : s . replace ( "<STR_LIT:_>" , "<STR_LIT>" ) ) ) <EOL> usage = "<STR_LIT>" % commands_str + "<STR_LIT>" % version_str <EOL> parser = optparse . OptionParser ( usage = usage ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = None ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = None ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = None ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT:date>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = None ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT:user>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT:message>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = None ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT:all>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT:version>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' % Mode . values ( ) , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:string>' , default = None ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' % Mode . PARTIAL , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' ) <EOL> parser . add_option ( '<STR_LIT>' , help = '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store>' , type = '<STR_LIT:int>' , default = <NUM_LIT:1> ) <EOL> ( options , args ) = parser . parse_args ( ) <EOL> if options . version : <EOL> print "<STR_LIT:%s>" % version_str <EOL> sys . exit ( <NUM_LIT:0> ) <EOL> if options . debug : <EOL> log_setup ( verbosity = <NUM_LIT:3> ) <EOL> log . debug ( "<STR_LIT>" , version_str ) <EOL> elif options . verbose : <EOL> log_setup ( verbosity = <NUM_LIT:2> ) <EOL> elif options . minimal : <EOL> log_setup ( verbosity = <NUM_LIT:0> ) <EOL> else : <EOL> log_setup ( verbosity = options . verbosity ) <EOL> if len ( args ) < <NUM_LIT:1> : <EOL> fail ( "<STR_LIT>" ) <EOL> command = args [ <NUM_LIT:0> ] <EOL> command_args = args [ <NUM_LIT:1> : ] <EOL> if command not in _COMMAND_LIST_REPO and command not in _COMMAND_LIST_WORK and command not in _COMMAND_ABBREVS : <EOL> fail ( "<STR_LIT>" ) <EOL> try : <EOL> run_command ( command , command_args , options ) <EOL> except InvalidArgument , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> fail ( str ( e ) , prefix = "<STR_LIT>" , exc_info = e , suppress = ( options . suppress and e . suppressable ) ) <EOL> except InvalidOperation , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> fail ( str ( e ) , prefix = "<STR_LIT>" , exc_info = e , suppress = ( options . suppress and e . suppressable ) ) <EOL> except Failure , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> fail ( str ( e ) , prefix = "<STR_LIT:error>" , exc_info = e , suppress = ( options . suppress and e . suppressable ) ) <EOL> except AssertionError , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> fail ( "<STR_LIT>" % e . message , exc_info = e ) <EOL> except boto . exception . S3ResponseError , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> fail ( str ( e ) , prefix = "<STR_LIT>" , exc_info = e ) <EOL> except IOError , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> if e . errno == errno . EPIPE : <EOL> fail ( status = <NUM_LIT:1> ) <EOL> else : <EOL> fail ( str ( e ) , status = <NUM_LIT:1> ) <EOL> except KeyboardInterrupt , e : <EOL> log . debug ( "<STR_LIT>" ) <EOL> fail ( status = <NUM_LIT:2> ) <EOL> except Exception , e : <EOL> log . debug ( "<STR_LIT>" , command , exc_info = e ) <EOL> fail ( "<STR_LIT>" % ( type ( e ) . __name__ , e ) , exc_info = e ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
