<s> """<STR_LIT>""" <EOL> import os <EOL> import time <EOL> from os . path import expanduser , join as opj , exists , isabs , lexists , curdir , realpath <EOL> from os . path import split as ops <EOL> from os . path import isdir , islink <EOL> from os import unlink , makedirs <EOL> from collections import OrderedDict <EOL> from humanize import naturalsize <EOL> from six import iteritems <EOL> from six import string_types <EOL> from distutils . version import LooseVersion <EOL> from git import Repo <EOL> from ... version import __version__ <EOL> from ... api import add_archive_content <EOL> from ... api import clean <EOL> from ... consts import CRAWLER_META_DIR , CRAWLER_META_CONFIG_FILENAME <EOL> from ... utils import rmtree , updated <EOL> from ... utils import lmtime <EOL> from ... utils import find_files <EOL> from ... utils import auto_repr <EOL> from ... tests . utils import put_file_under_git <EOL> from ... downloaders . providers import Providers <EOL> from ... support . configparserinc import SafeConfigParserWithIncludes <EOL> from ... support . gitrepo import GitRepo , _normalize_path <EOL> from ... support . annexrepo import AnnexRepo <EOL> from ... support . handlerepo import HandleRepo <EOL> from ... support . stats import ActivityStats <EOL> from ... support . versions import get_versions <EOL> from ... support . network import get_url_straight_filename , get_url_disposition_filename <EOL> from ... import cfg <EOL> from ... cmd import get_runner <EOL> from . . pipeline import CRAWLER_PIPELINE_SECTION <EOL> from . . dbs . files import PhysicalFileStatusesDB , JsonFileStatusesDB <EOL> from . . dbs . versions import SingleVersionDB <EOL> from logging import getLogger <EOL> lgr = getLogger ( '<STR_LIT>' ) <EOL> _runner = get_runner ( ) <EOL> _call = _runner . call <EOL> _run = _runner . run <EOL> @ auto_repr <EOL> class initiate_handle ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , template , handle_name = None , collection_name = None , <EOL> path = None , branch = None , backend = None , <EOL> data_fields = [ ] , add_fields = { } , existing = None ) : <EOL> """<STR_LIT>""" <EOL> self . template = template <EOL> self . handle_name = handle_name <EOL> self . collection_name = collection_name <EOL> self . data_fields = data_fields <EOL> self . add_fields = add_fields <EOL> self . existing = existing <EOL> self . path = path <EOL> self . branch = branch <EOL> self . backend = backend <EOL> def _initiate_handle ( self , path , name ) : <EOL> lgr . info ( "<STR_LIT>" % name ) <EOL> if self . branch is not None : <EOL> git_repo = GitRepo ( path , create = True ) <EOL> git_repo . git_checkout ( self . branch , options = "<STR_LIT>" ) <EOL> backend = self . backend or cfg . get ( '<STR_LIT>' , '<STR_LIT>' , default = '<STR_LIT>' ) <EOL> repo = HandleRepo ( <EOL> path , <EOL> direct = cfg . getboolean ( '<STR_LIT>' , '<STR_LIT>' , default = False ) , <EOL> name = name , <EOL> backend = backend , <EOL> create = True ) <EOL> if backend : <EOL> put_file_under_git ( path , '<STR_LIT>' , '<STR_LIT>' % backend , annexed = False ) <EOL> return repo <EOL> def _save_crawl_config ( self , handle_path , name , data ) : <EOL> lgr . debug ( "<STR_LIT>" % name ) <EOL> repo = GitRepo ( handle_path ) <EOL> crawl_config_dir = opj ( handle_path , CRAWLER_META_DIR ) <EOL> if not exists ( crawl_config_dir ) : <EOL> lgr . log ( <NUM_LIT:2> , "<STR_LIT>" , crawl_config_dir ) <EOL> makedirs ( crawl_config_dir ) <EOL> crawl_config = opj ( crawl_config_dir , CRAWLER_META_CONFIG_FILENAME ) <EOL> cfg = SafeConfigParserWithIncludes ( ) <EOL> cfg . add_section ( CRAWLER_PIPELINE_SECTION ) <EOL> def secset ( k , v ) : <EOL> cfg . set ( CRAWLER_PIPELINE_SECTION , k , str ( v ) ) <EOL> secset ( '<STR_LIT>' , self . template ) <EOL> for f in self . data_fields : <EOL> secset ( f , data [ f ] ) <EOL> for k , v in self . add_fields : <EOL> secset ( k , v ) <EOL> with open ( crawl_config , '<STR_LIT:w>' ) as f : <EOL> cfg . write ( f ) <EOL> repo . git_add ( crawl_config ) <EOL> if repo . dirty : <EOL> repo . git_commit ( "<STR_LIT>" % self . template ) <EOL> else : <EOL> lgr . debug ( "<STR_LIT>" ) <EOL> def __call__ ( self , data = { } ) : <EOL> handle_name = self . handle_name or data [ '<STR_LIT>' ] <EOL> if self . path is None : <EOL> crawl_toppath = cfg . get ( '<STR_LIT>' , '<STR_LIT>' , <EOL> default = opj ( expanduser ( '<STR_LIT>' ) , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> handle_path = opj ( crawl_toppath , <EOL> self . collection_name or self . template , <EOL> handle_name ) <EOL> else : <EOL> handle_path = self . path <EOL> lgr . debug ( "<STR_LIT>" , handle_path ) <EOL> init = True <EOL> if exists ( handle_path ) : <EOL> existing = self . existing or '<STR_LIT>' <EOL> if existing == '<STR_LIT>' : <EOL> lgr . info ( "<STR_LIT>" % handle_name ) <EOL> yield data <EOL> return <EOL> elif existing == '<STR_LIT>' : <EOL> raise RuntimeError ( "<STR_LIT>" % handle_path ) <EOL> elif existing == '<STR_LIT:replace>' : <EOL> _call ( rmtree , handle_path ) <EOL> elif existing == '<STR_LIT>' : <EOL> init = False <EOL> else : <EOL> raise ValueError ( self . existing ) <EOL> if init : <EOL> _call ( self . _initiate_handle , handle_path , handle_name ) <EOL> _call ( self . _save_crawl_config , handle_path , handle_name , data ) <EOL> yield updated ( data , { '<STR_LIT>' : handle_path , <EOL> '<STR_LIT>' : handle_name } ) <EOL> class Annexificator ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , path = None , mode = '<STR_LIT>' , options = None , <EOL> special_remotes = [ ] , <EOL> allow_dirty = False , yield_non_updated = False , <EOL> auto_finalize = True , <EOL> statusdb = None , <EOL> ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if path is None : <EOL> path = realpath ( curdir ) <EOL> self . repo = AnnexRepo ( path , always_commit = False , ** kwargs ) <EOL> git_remotes = self . repo . git_get_remotes ( ) <EOL> if special_remotes : <EOL> for remote in special_remotes : <EOL> if remote not in git_remotes : <EOL> self . repo . annex_initremote ( <EOL> remote , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' % remote ] ) <EOL> self . mode = mode <EOL> self . options = options or [ ] <EOL> self . auto_finalize = auto_finalize <EOL> self . _states = set ( ) <EOL> self . _providers = Providers . from_config_files ( ) <EOL> self . yield_non_updated = yield_non_updated <EOL> if ( not allow_dirty ) and self . repo . dirty : <EOL> raise RuntimeError ( "<STR_LIT>" % path ) <EOL> self . statusdb = statusdb <EOL> self . _statusdb = None <EOL> def register_url_in_db ( self , url , filename ) : <EOL> raise NotImplementedError ( ) <EOL> def reset ( self ) : <EOL> if self . _statusdb : <EOL> self . _statusdb . reset ( ) <EOL> @ staticmethod <EOL> def _get_filename_from_url ( url ) : <EOL> if url is None : <EOL> return None <EOL> filename = get_url_straight_filename ( url ) <EOL> if not filename : <EOL> filename = get_url_disposition_filename ( url ) <EOL> if not filename : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" % url ) <EOL> return filename <EOL> def __call__ ( self , data ) : <EOL> assert ( self . mode is not None ) <EOL> stats = data . get ( '<STR_LIT>' , ActivityStats ( ) ) <EOL> url = data . get ( '<STR_LIT:url>' ) <EOL> if url : <EOL> stats . urls += <NUM_LIT:1> <EOL> fpath = self . _get_fpath ( data , stats , url ) <EOL> filepath = opj ( self . repo . path , fpath ) <EOL> lgr . debug ( "<STR_LIT>" , locals ( ) ) <EOL> updated_data = updated ( data , { '<STR_LIT:filename>' : ops ( fpath ) [ <NUM_LIT:1> ] , <EOL> } ) <EOL> remote_status = None <EOL> if self . statusdb is not None and self . _statusdb is None : <EOL> if isinstance ( self . statusdb , string_types ) : <EOL> self . _statusdb = { <EOL> '<STR_LIT>' : JsonFileStatusesDB , <EOL> '<STR_LIT>' : PhysicalFileStatusesDB } [ self . statusdb ] ( annex = self . repo ) <EOL> else : <EOL> self . _statusdb = self . statusdb <EOL> statusdb = self . _statusdb <EOL> if url : <EOL> downloader = self . _providers . get_provider ( url ) . get_downloader ( url ) <EOL> remote_status = data [ '<STR_LIT>' ] if '<STR_LIT>' in data else downloader . get_status ( url ) <EOL> if lexists ( filepath ) : <EOL> if self . mode == '<STR_LIT>' or ( statusdb is not None and not statusdb . is_different ( fpath , remote_status , url ) ) : <EOL> lgr . debug ( "<STR_LIT>" , <EOL> url , filepath , remote_status ) <EOL> stats . skipped += <NUM_LIT:1> <EOL> if self . yield_non_updated : <EOL> yield updated_data <EOL> return <EOL> else : <EOL> if statusdb : <EOL> statusdb . get ( fpath ) <EOL> if not url : <EOL> lgr . debug ( "<STR_LIT>" % ( filepath ) ) <EOL> assert ( fpath ) <EOL> out_json = _call ( self . repo . annex_add , fpath , options = self . options ) <EOL> else : <EOL> annex_options = self . options <EOL> if self . mode == '<STR_LIT>' : <EOL> lgr . debug ( "<STR_LIT>" % ( url , filepath ) ) <EOL> else : <EOL> annex_options = annex_options + [ "<STR_LIT>" % self . mode ] <EOL> lgr . debug ( "<STR_LIT>" % ( url , filepath , self . mode ) ) <EOL> if lexists ( filepath ) : <EOL> lgr . debug ( "<STR_LIT>" % filepath ) <EOL> if isdir ( filepath ) : <EOL> _call ( self . _check_no_staged_changes_under_dir , filepath , stats = stats ) <EOL> _call ( rmtree , filepath ) <EOL> else : <EOL> _call ( unlink , filepath ) <EOL> _call ( stats . increment , '<STR_LIT>' ) <EOL> else : <EOL> _call ( self . _check_non_existing_filepath , filepath , stats = stats ) <EOL> if self . mode == '<STR_LIT>' and remote_status and remote_status . size : <EOL> lgr . info ( "<STR_LIT>" <EOL> % ( naturalsize ( remote_status . size ) , url ) ) <EOL> out_json = _call ( self . repo . annex_addurl_to_file , fpath , url , options = annex_options , batch = True ) <EOL> added_to_annex = '<STR_LIT:key>' in out_json <EOL> if self . mode == '<STR_LIT>' or not added_to_annex : <EOL> _call ( stats . increment , '<STR_LIT>' ) <EOL> _call ( stats . increment , '<STR_LIT>' , _call ( lambda : os . stat ( filepath ) . st_size ) ) <EOL> _call ( stats . increment , '<STR_LIT>' if '<STR_LIT:key>' in out_json else '<STR_LIT>' ) <EOL> if lexists ( filepath ) : <EOL> if remote_status : <EOL> if remote_status . mtime : <EOL> _call ( lmtime , filepath , remote_status . mtime ) <EOL> if statusdb : <EOL> _call ( statusdb . set , filepath , remote_status ) <EOL> else : <EOL> if statusdb : <EOL> _call ( statusdb . set , filepath ) <EOL> self . _states . add ( "<STR_LIT>" ) <EOL> yield updated_data <EOL> def _check_no_staged_changes_under_dir ( self , dirpath , stats = None ) : <EOL> """<STR_LIT>""" <EOL> dirty_files = self . _get_status ( ) <EOL> dirty_files = sum ( dirty_files , [ ] ) <EOL> dirpath_normalized = _normalize_path ( self . repo . path , dirpath ) <EOL> for dirty_file in dirty_files : <EOL> if stats : <EOL> _call ( stats . increment , '<STR_LIT>' ) <EOL> if dirty_file . startswith ( dirpath_normalized ) : <EOL> if self . auto_finalize : <EOL> self . finalize ( ) ( { '<STR_LIT>' : stats } ) <EOL> return <EOL> else : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % locals ( ) ) <EOL> def _check_non_existing_filepath ( self , filepath , stats = None ) : <EOL> """<STR_LIT>""" <EOL> dirpath , name = ops ( filepath ) <EOL> if dirpath : <EOL> while dirpath : <EOL> if lexists ( dirpath ) : <EOL> if not isdir ( dirpath ) : <EOL> dirty_files = self . _get_status ( ) <EOL> dirty_files = sum ( dirty_files , [ ] ) <EOL> dirpath_normalized = _normalize_path ( self . repo . path , dirpath ) <EOL> if dirpath_normalized in dirty_files : <EOL> if self . auto_finalize : <EOL> self . finalize ( ) ( { '<STR_LIT>' : stats } ) <EOL> else : <EOL> raise RuntimeError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % locals ( ) ) <EOL> lgr . debug ( "<STR_LIT>" % ( dirpath , filepath ) ) <EOL> _call ( os . unlink , dirpath ) <EOL> if stats : <EOL> _call ( stats . increment , '<STR_LIT>' ) <EOL> break <EOL> dirpath , _ = ops ( dirpath ) <EOL> if not dirpath . startswith ( self . repo . path ) : <EOL> raise RuntimeError ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( dirpath , self . repo . path ) ) <EOL> def _get_fpath ( self , data , stats , url = None ) : <EOL> """<STR_LIT>""" <EOL> fpath = filename = data [ '<STR_LIT:filename>' ] if '<STR_LIT:filename>' in data else self . _get_filename_from_url ( url ) <EOL> stats . files += <NUM_LIT:1> <EOL> if filename is None : <EOL> stats . skipped += <NUM_LIT:1> <EOL> raise ValueError ( "<STR_LIT>" % url ) <EOL> elif isabs ( filename ) : <EOL> stats . skipped += <NUM_LIT:1> <EOL> raise ValueError ( "<STR_LIT>" % filename ) <EOL> path_ = data . get ( '<STR_LIT:path>' , None ) <EOL> if path_ : <EOL> if isabs ( path_ ) : <EOL> stats . skipped += <NUM_LIT:1> <EOL> raise ValueError ( "<STR_LIT>" % path_ ) <EOL> fpath = opj ( path_ , fpath ) <EOL> return fpath <EOL> def switch_branch ( self , branch , parent = None ) : <EOL> """<STR_LIT>""" <EOL> def switch_branch ( data ) : <EOL> """<STR_LIT>""" % branch <EOL> list ( self . finalize ( ) ( data ) ) <EOL> self . _statusdb = None <EOL> existing_branches = self . repo . git_get_branches ( ) <EOL> if branch not in existing_branches : <EOL> if parent is None : <EOL> lgr . info ( "<STR_LIT>" % ( branch ) ) <EOL> self . repo . git_checkout ( branch , options = "<STR_LIT>" ) <EOL> if self . repo . dirty : <EOL> self . repo . git_remove ( '<STR_LIT:.>' , r = True , f = True ) <EOL> else : <EOL> if parent not in existing_branches : <EOL> raise RuntimeError ( "<STR_LIT>" % parent ) <EOL> lgr . info ( "<STR_LIT>" % ( parent , branch ) ) <EOL> self . repo . git_checkout ( parent , options = "<STR_LIT>" % branch ) <EOL> else : <EOL> lgr . info ( "<STR_LIT>" % ( branch ) ) <EOL> self . repo . git_checkout ( branch ) <EOL> yield updated ( data , { "<STR_LIT>" : branch } ) <EOL> return switch_branch <EOL> def merge_branch ( self , branch , target_branch = None , <EOL> strategy = None , commit = True , one_commit_at_a_time = False , skip_no_changes = None ) : <EOL> """<STR_LIT>""" <EOL> assert ( strategy in ( None , '<STR_LIT>' ) ) <EOL> def merge_branch ( data ) : <EOL> if target_branch is not None : <EOL> orig_branch = self . repo . git_get_active_branch ( ) <EOL> target_branch_ = target_branch <EOL> list ( self . switch_branch ( target_branch_ ) ( data ) ) <EOL> else : <EOL> orig_branch = None <EOL> target_branch_ = self . repo . git_get_active_branch ( ) <EOL> if self . repo . dirty : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> last_merged_checksum = self . repo . git_get_merge_base ( [ target_branch_ , branch ] ) <EOL> if last_merged_checksum == self . repo . git_get_hexsha ( branch ) : <EOL> lgr . debug ( "<STR_LIT>" % branch ) <EOL> skip_no_changes_ = skip_no_changes <EOL> if skip_no_changes is None : <EOL> skip_no_changes_ = True <EOL> if skip_no_changes_ : <EOL> lgr . debug ( "<STR_LIT>" ) <EOL> return <EOL> if one_commit_at_a_time : <EOL> all_to_merge = list ( <EOL> self . repo . git_get_branch_commits ( <EOL> branch , <EOL> limit = '<STR_LIT>' , <EOL> stop = last_merged_checksum , <EOL> value = '<STR_LIT>' ) ) [ : : - <NUM_LIT:1> ] <EOL> else : <EOL> all_to_merge = [ branch ] <EOL> nmerges = len ( all_to_merge ) <EOL> plmerges = "<STR_LIT:s>" if nmerges > <NUM_LIT:1> else "<STR_LIT>" <EOL> lgr . info ( "<STR_LIT>" , locals ( ) ) <EOL> options = [ '<STR_LIT>' ] if not commit else [ ] <EOL> for to_merge in all_to_merge : <EOL> if self . repo . git_get_active_branch ( ) != target_branch_ : <EOL> self . repo . git_checkout ( target_branch_ ) <EOL> if strategy is None : <EOL> self . repo . git_merge ( to_merge , options = options ) <EOL> elif strategy == '<STR_LIT>' : <EOL> self . repo . git_merge ( to_merge , options = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] , expect_stderr = True ) <EOL> self . repo . _git_custom_command ( [ ] , "<STR_LIT>" % to_merge ) <EOL> self . repo . annex_add ( '<STR_LIT:.>' , options = self . options ) <EOL> else : <EOL> raise NotImplementedError ( strategy ) <EOL> if commit : <EOL> if strategy is not None : <EOL> msg = branch if ( nmerges == <NUM_LIT:1> ) else ( "<STR_LIT>" % ( branch , to_merge ) ) <EOL> self . _commit ( "<STR_LIT>" % ( msg , strategy ) , options = [ "<STR_LIT>" ] ) <EOL> else : <EOL> stats = data . get ( '<STR_LIT>' , None ) <EOL> if stats : <EOL> stats . merges . append ( [ branch , target_branch_ ] ) <EOL> if orig_branch is not None : <EOL> self . repo . git_checkout ( orig_branch ) <EOL> yield data <EOL> return merge_branch <EOL> def _precommit ( self ) : <EOL> self . repo . precommit ( ) <EOL> if self . _statusdb : <EOL> self . _statusdb . save ( ) <EOL> path = self . repo . path <EOL> if self . repo . dirty and not exists ( opj ( path , '<STR_LIT>' ) ) : <EOL> backends = self . repo . default_backends <EOL> if backends : <EOL> put_file_under_git ( path , '<STR_LIT>' , '<STR_LIT>' % backends [ <NUM_LIT:0> ] , <EOL> annexed = False ) <EOL> def _commit ( self , msg = None , options = [ ] ) : <EOL> if msg is not None : <EOL> options = options + [ "<STR_LIT>" , msg ] <EOL> self . _precommit ( ) <EOL> self . repo . _git_custom_command ( [ ] , [ "<STR_LIT>" , "<STR_LIT>" ] + options ) <EOL> def _unstage ( self , fpaths ) : <EOL> self . repo . _git_custom_command ( fpaths , [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> def _stage ( self , fpaths ) : <EOL> self . repo . git_add ( fpaths ) <EOL> def _get_status ( self ) : <EOL> """<STR_LIT>""" <EOL> out , err = self . repo . _git_custom_command ( [ ] , [ "<STR_LIT>" , "<STR_LIT:status>" , "<STR_LIT>" ] ) <EOL> assert not err <EOL> staged , notstaged , untracked , deleted = [ ] , [ ] , [ ] , [ ] <EOL> for l in out . split ( '<STR_LIT:\n>' ) : <EOL> if not l : <EOL> continue <EOL> act = l [ : <NUM_LIT:2> ] <EOL> fname = l [ <NUM_LIT:3> : ] <EOL> try : <EOL> { '<STR_LIT>' : untracked , <EOL> '<STR_LIT>' : staged , <EOL> '<STR_LIT>' : staged , <EOL> '<STR_LIT>' : notstaged , <EOL> '<STR_LIT>' : deleted , <EOL> } [ act ] . append ( fname ) <EOL> except KeyError : <EOL> raise RuntimeError ( "<STR_LIT>" % act ) <EOL> return staged , notstaged , untracked , deleted <EOL> def commit_versions ( self , <EOL> regex , <EOL> dirs = True , <EOL> rename = False , <EOL> ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> def _commit_versions ( data ) : <EOL> self . _precommit ( ) <EOL> staged , notstaged , untracked , deleted = self . _get_status ( ) <EOL> assert ( not notstaged ) <EOL> assert ( not untracked ) <EOL> assert ( not deleted ) <EOL> if not staged : <EOL> return <EOL> if not dirs : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> versions = get_versions ( staged , regex , ** kwargs ) <EOL> if not versions : <EOL> for d in self . finalize ( ) ( data ) : <EOL> yield d <EOL> return <EOL> if None in versions : <EOL> versions . pop ( None ) <EOL> versions_db = SingleVersionDB ( self . repo ) <EOL> prev_version = versions_db . version <EOL> if prev_version is None : <EOL> new_versions = versions <EOL> else : <EOL> version_keys = list ( versions . keys ( ) ) <EOL> if prev_version not in versions_db . versions : <EOL> raise RuntimeError ( "<STR_LIT>" % ( prev_version , versions_db . versions . keys ( ) ) ) <EOL> assert ( all ( ( LooseVersion ( prev_version ) < LooseVersion ( v ) ) for v in versions ) ) <EOL> new_versions = versions <EOL> if new_versions : <EOL> smallest_new_version = next ( iter ( new_versions ) ) <EOL> if prev_version : <EOL> if LooseVersion ( smallest_new_version ) <= LooseVersion ( prev_version ) : <EOL> raise ValueError ( "<STR_LIT>" <EOL> % ( smallest_new_version , prev_version ) ) <EOL> versions_db . update_versions ( versions ) <EOL> nnew_versions = len ( new_versions ) <EOL> if nnew_versions <= <NUM_LIT:1> : <EOL> if nnew_versions == <NUM_LIT:1> : <EOL> _call ( setattr , versions_db , '<STR_LIT:version>' , smallest_new_version ) <EOL> for d in self . finalize ( ) ( data ) : <EOL> yield d <EOL> return <EOL> nunstaged = <NUM_LIT:0> <EOL> for version , fpaths in iteritems ( versions ) : <EOL> nfpaths = len ( fpaths ) <EOL> lgr . debug ( "<STR_LIT>" , nfpaths , version ) <EOL> nunstaged += nfpaths <EOL> _call ( self . _unstage , list ( fpaths . values ( ) ) ) <EOL> stats = data . get ( '<STR_LIT>' , None ) <EOL> stats_str = ( '<STR_LIT>' + stats . as_str ( mode = '<STR_LIT>' ) ) if stats else '<STR_LIT>' <EOL> for iversion , ( version , fpaths ) in enumerate ( iteritems ( new_versions ) ) : <EOL> if rename : <EOL> vfpaths = [ ] <EOL> for fpath , vfpath in iteritems ( fpaths ) : <EOL> lgr . debug ( "<STR_LIT>" % ( vfpath , fpath ) ) <EOL> os . rename ( vfpath , fpath ) <EOL> vfpaths . append ( fpath ) <EOL> else : <EOL> vfpaths = list ( fpaths . values ( ) ) <EOL> nfpaths = len ( vfpaths ) <EOL> lgr . debug ( "<STR_LIT>" , nfpaths , version ) <EOL> nunstaged -= nfpaths <EOL> assert ( nfpaths >= <NUM_LIT:0> ) <EOL> assert ( nunstaged >= <NUM_LIT:0> ) <EOL> _call ( self . _stage , vfpaths ) <EOL> vmsg = "<STR_LIT>" % ( iversion + <NUM_LIT:1> , nnew_versions , version , nunstaged ) <EOL> if stats : <EOL> _call ( stats . reset ) <EOL> if version : <EOL> _call ( setattr , versions_db , '<STR_LIT:version>' , version ) <EOL> _call ( self . _commit , "<STR_LIT>" % ( '<STR_LIT:U+002CU+0020>' . join ( self . _states ) , vmsg , stats_str ) , options = [ ] ) <EOL> yield data <EOL> assert ( nunstaged == <NUM_LIT:0> ) <EOL> return _commit_versions <EOL> def remove_other_versions ( self , name = None , overlay = False ) : <EOL> """<STR_LIT>""" <EOL> def _remove_other_versions ( data ) : <EOL> if overlay : <EOL> raise NotImplementedError ( overlay ) <EOL> stats = data . get ( '<STR_LIT>' , None ) <EOL> versions_db = SingleVersionDB ( self . repo , name = name ) <EOL> current_version = versions_db . version <EOL> if not current_version : <EOL> lgr . info ( "<STR_LIT>" ) <EOL> yield data <EOL> return <EOL> for version , fpaths in iteritems ( versions_db . versions ) : <EOL> if version is None or current_version == version : <EOL> continue <EOL> for fpath , vfpath in iteritems ( fpaths ) : <EOL> vfpathfull = opj ( self . repo . path , vfpath ) <EOL> if lexists ( vfpathfull ) : <EOL> lgr . log ( <NUM_LIT:5> , "<STR_LIT>" , vfpathfull , version , current_version ) <EOL> os . unlink ( vfpathfull ) <EOL> if stats : <EOL> stats . versions . append ( current_version ) <EOL> yield data <EOL> return _remove_other_versions <EOL> def add_archive_content ( self , commit = False , ** aac_kwargs ) : <EOL> """<STR_LIT>""" <EOL> def _add_archive_content ( data ) : <EOL> stats = data . get ( '<STR_LIT>' , ActivityStats ( ) ) <EOL> archive = self . _get_fpath ( data , stats ) <EOL> annex = add_archive_content ( <EOL> archive , annex = self . repo , <EOL> delete = True , key = False , commit = commit , allow_dirty = True , <EOL> annex_options = self . options , <EOL> stats = stats , <EOL> ** aac_kwargs <EOL> ) <EOL> self . _states . add ( "<STR_LIT>" ) <EOL> assert ( annex is self . repo ) <EOL> yield updated ( data , { '<STR_LIT>' : stats } ) <EOL> return _add_archive_content <EOL> def finalize ( self , tag = False , existing_tag = None , cleanup = False ) : <EOL> """<STR_LIT>""" <EOL> def _finalize ( data ) : <EOL> self . _precommit ( ) <EOL> stats = data . get ( '<STR_LIT>' , None ) <EOL> if self . repo . dirty : <EOL> lgr . info ( "<STR_LIT>" ) <EOL> _call ( self . repo . annex_add , '<STR_LIT:.>' , options = self . options ) <EOL> stats_str = ( '<STR_LIT>' + stats . as_str ( mode = '<STR_LIT>' ) ) if stats else '<STR_LIT>' <EOL> _call ( self . _commit , "<STR_LIT>" % ( '<STR_LIT:U+002CU+0020>' . join ( self . _states ) , stats_str ) , options = [ "<STR_LIT>" ] ) <EOL> if stats : <EOL> _call ( stats . reset ) <EOL> else : <EOL> lgr . info ( "<STR_LIT>" ) <EOL> if tag and stats : <EOL> total_stats = stats . get_total ( ) <EOL> if total_stats . versions : <EOL> last_version = total_stats . versions [ - <NUM_LIT:1> ] <EOL> if isinstance ( tag , string_types ) : <EOL> tag_ = tag . format ( stats = total_stats , data = data , last_version = last_version ) <EOL> else : <EOL> tag_ = last_version <EOL> stats_str = "<STR_LIT>" + total_stats . as_str ( mode = '<STR_LIT>' ) <EOL> tags = self . repo . repo . tags <EOL> if tag_ in tags : <EOL> if existing_tag == '<STR_LIT>' : <EOL> lgr . warning ( "<STR_LIT>" % tag_ ) <EOL> tag_ = None <EOL> elif existing_tag is None : <EOL> suf = <NUM_LIT:1> <EOL> while True : <EOL> tag__ = '<STR_LIT>' % ( tag_ , suf ) <EOL> if tag__ not in tags : <EOL> break <EOL> suf += <NUM_LIT:1> <EOL> lgr . warning ( "<STR_LIT>" % ( tag_ , tag__ ) ) <EOL> tag_ = tag__ <EOL> else : <EOL> raise ValueError ( existing_tag ) <EOL> self . repo . repo . create_tag ( tag_ , message = "<STR_LIT>" % ( __version__ , stats_str ) ) <EOL> if cleanup : <EOL> total_stats = stats . get_total ( ) <EOL> if total_stats . add_git or total_stats . add_annex or total_stats . merges : <EOL> if cfg . getboolean ( '<STR_LIT>' , '<STR_LIT>' , default = True ) : <EOL> lgr . info ( "<STR_LIT>" ) <EOL> self . repo . gc ( allow_background = False ) <EOL> clean ( annex = self . repo ) <EOL> else : <EOL> lgr . info ( "<STR_LIT>" ) <EOL> else : <EOL> lgr . info ( "<STR_LIT>" ) <EOL> self . _states = set ( ) <EOL> yield data <EOL> return _finalize <EOL> def remove_obsolete ( self ) : <EOL> """<STR_LIT>""" <EOL> class _remove_obsolete ( object ) : <EOL> def __call__ ( self_ , data ) : <EOL> statusdb = self . _statusdb <EOL> obsolete = statusdb . get_obsolete ( ) <EOL> if obsolete : <EOL> files_str = "<STR_LIT>" + '<STR_LIT:U+002CU+0020>' . join ( obsolete ) if len ( obsolete ) < <NUM_LIT:10> else "<STR_LIT>" <EOL> lgr . info ( '<STR_LIT>' % ( len ( obsolete ) , files_str ) ) <EOL> stats = data . get ( '<STR_LIT>' , None ) <EOL> _call ( self . repo . git_remove , obsolete ) <EOL> if stats : <EOL> _call ( stats . increment , '<STR_LIT>' , len ( obsolete ) ) <EOL> for filepath in obsolete : <EOL> statusdb . remove ( filepath ) <EOL> yield data <EOL> def reset ( self_ ) : <EOL> if self . _statusdb : <EOL> self . _statusdb . reset ( ) <EOL> return _remove_obsolete ( ) <EOL> def remove ( self , data ) : <EOL> """<STR_LIT>""" <EOL> stats = data . get ( '<STR_LIT>' , None ) <EOL> self . _states . add ( "<STR_LIT>" ) <EOL> filename = self . _get_fpath ( data , stats ) <EOL> if stats : <EOL> _call ( stats . increment , '<STR_LIT>' ) <EOL> _call ( self . repo . git_remove , filename ) <EOL> yield data </s>
