<s> import json <EOL> import sys , os <EOL> from os import path <EOL> import urllib2 <EOL> import uuid <EOL> import subprocess <EOL> from StringIO import StringIO <EOL> import re <EOL> from string import Template <EOL> from datetime import datetime , timedelta <EOL> import logging <EOL> import argparse <EOL> import dateutil . parser <EOL> import getpass <EOL> import pexpect <EOL> import expire , config <EOL> from config import Job <EOL> class ArgumentError ( Exception ) : <EOL> pass <EOL> class TarsnapError ( Exception ) : <EOL> pass <EOL> class TarsnapBackend ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , log , options , dryrun = False ) : <EOL> """<STR_LIT>""" <EOL> self . log = log <EOL> self . options = options <EOL> self . dryrun = dryrun <EOL> self . _queried_archives = None <EOL> self . _known_archives = [ ] <EOL> self . key_passphrase = None <EOL> def call ( self , * arguments ) : <EOL> """<STR_LIT>""" <EOL> call_with = [ '<STR_LIT>' ] <EOL> for option in self . options : <EOL> key = option [ <NUM_LIT:0> ] <EOL> pre = "<STR_LIT:->" if len ( key ) == <NUM_LIT:1> else "<STR_LIT>" <EOL> call_with . append ( "<STR_LIT>" % ( pre , key ) ) <EOL> for value in option [ <NUM_LIT:1> : ] : <EOL> call_with . append ( value ) <EOL> call_with . extend ( arguments ) <EOL> return self . _exec_tarsnap ( call_with ) <EOL> def _exec_tarsnap ( self , args ) : <EOL> self . log . debug ( "<STR_LIT>" % "<STR_LIT:U+0020>" . join ( args ) ) <EOL> env = os . environ <EOL> env [ '<STR_LIT>' ] = '<STR_LIT:C>' <EOL> child = pexpect . spawn ( args [ <NUM_LIT:0> ] , args [ <NUM_LIT:1> : ] , env = env , timeout = None ) <EOL> if self . log . isEnabledFor ( logging . DEBUG ) : <EOL> child . logfile = sys . stdout <EOL> has_prompt = ( child . expect ( [ '<STR_LIT>' , pexpect . EOF ] ) == <NUM_LIT:0> ) <EOL> if has_prompt : <EOL> child . sendline ( self . _get_key_passphrase ( ) ) <EOL> child . expect ( pexpect . EOF ) <EOL> out = child . before <EOL> child . close ( ) <EOL> if child . exitstatus != <NUM_LIT:0> : <EOL> raise TarsnapError ( "<STR_LIT>" . format ( <EOL> child . exitstatus , os . linesep , out ) ) <EOL> return out <EOL> def _get_key_passphrase ( self ) : <EOL> if not self . key_passphrase : <EOL> self . key_passphrase = getpass . getpass ( '<STR_LIT>' ) <EOL> return self . key_passphrase <EOL> def _exec_util ( self , cmdline , shell = False ) : <EOL> self . log . debug ( "<STR_LIT>" % cmdline ) <EOL> p = subprocess . Popen ( cmdline , shell = True ) <EOL> p . communicate ( ) <EOL> if p . returncode : <EOL> raise RuntimeError ( '<STR_LIT>' % ( <EOL> cmdline , p . returncode ) ) <EOL> def _add_known_archive ( self , name ) : <EOL> """<STR_LIT>""" <EOL> self . _known_archives . append ( name ) <EOL> def get_archives ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _queried_archives is None : <EOL> response = StringIO ( self . call ( '<STR_LIT>' ) ) <EOL> self . _queried_archives = [ l . rstrip ( ) for l in response . readlines ( ) ] <EOL> if [ '<STR_LIT:v>' ] in self . options : <EOL> self . _queried_archives = [ <EOL> l . rsplit ( '<STR_LIT:\t>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] for l in self . _queried_archives ] <EOL> return self . _queried_archives + self . _known_archives <EOL> archives = property ( get_archives ) <EOL> def get_backups ( self , job ) : <EOL> """<STR_LIT>""" <EOL> unique = uuid . uuid4 ( ) . hex <EOL> regexes = [ ] <EOL> for possible_name in [ job . name ] + ( job . aliases or [ ] ) : <EOL> target = Template ( job . target ) . substitute ( <EOL> { '<STR_LIT:name>' : possible_name , '<STR_LIT:date>' : unique } ) <EOL> regexes . append ( re . compile ( "<STR_LIT>" % <EOL> re . escape ( target ) . replace ( unique , '<STR_LIT>' ) ) ) <EOL> backups = { } <EOL> for backup_path in self . get_archives ( ) : <EOL> match = None <EOL> for regex in regexes : <EOL> match = regex . match ( backup_path ) <EOL> if match : <EOL> break <EOL> else : <EOL> continue <EOL> try : <EOL> date = parse_date ( match . groupdict ( ) [ '<STR_LIT:date>' ] , job . dateformat ) <EOL> except ValueError , e : <EOL> self . log . error ( "<STR_LIT>" % ( backup_path , e ) ) <EOL> else : <EOL> backups [ backup_path ] = date <EOL> return backups <EOL> def expire ( self , job ) : <EOL> """<STR_LIT>""" <EOL> backups = self . get_backups ( job ) <EOL> self . log . info ( '<STR_LIT>' % len ( backups ) ) <EOL> to_keep = expire . expire ( backups , job . deltas ) <EOL> self . log . info ( '<STR_LIT>' % ( len ( backups ) - len ( to_keep ) ) ) <EOL> to_delete = [ ] <EOL> for name , _ in backups . items ( ) : <EOL> if not name in to_keep : <EOL> to_delete . append ( name ) <EOL> to_keep . sort ( ) <EOL> to_delete . sort ( ) <EOL> self . log . debug ( '<STR_LIT>' % '<STR_LIT:U+0020>' . join ( to_keep ) ) <EOL> if len ( to_delete ) == <NUM_LIT:0> : <EOL> return <EOL> self . log . info ( '<STR_LIT>' % '<STR_LIT:U+0020>' . join ( to_delete ) ) <EOL> if not self . dryrun : <EOL> self . call ( '<STR_LIT>' , '<STR_LIT>' , * '<STR_LIT>' . join ( to_delete ) . split ( '<STR_LIT:U+0020>' ) ) <EOL> for name in to_delete : <EOL> self . archives . remove ( name ) <EOL> def make ( self , job ) : <EOL> now = datetime . utcnow ( ) <EOL> date_str = now . strftime ( job . dateformat or DEFAULT_DATEFORMAT ) <EOL> target = Template ( job . target ) . safe_substitute ( <EOL> { '<STR_LIT:date>' : date_str , '<STR_LIT:name>' : job . name } ) <EOL> if job . name : <EOL> self . log . info ( '<STR_LIT>' % ( job . name , target ) ) <EOL> else : <EOL> self . log . info ( '<STR_LIT>' % target ) <EOL> if not self . dryrun : <EOL> args = [ '<STR_LIT:-c>' ] <EOL> [ args . extend ( [ '<STR_LIT>' , e ] ) for e in job . excludes ] <EOL> args . extend ( [ '<STR_LIT>' , target ] ) <EOL> args . extend ( job . sources ) <EOL> self . call ( * args ) <EOL> self . _add_known_archive ( target ) <EOL> return target , now <EOL> DEFAULT_DATEFORMAT = '<STR_LIT>' <EOL> def parse_date ( string , dateformat = None ) : <EOL> """<STR_LIT>""" <EOL> if dateformat : <EOL> return datetime . strptime ( string , dateformat ) <EOL> else : <EOL> return dateutil . parser . parse ( string ) <EOL> def timedelta_string ( value ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return config . str_to_timedelta ( value ) <EOL> except ValueError , e : <EOL> raise argparse . ArgumentTypeError ( '<STR_LIT>' % e ) <EOL> class Command ( object ) : <EOL> BackendClass = TarsnapBackend <EOL> def __init__ ( self , args , log , backend_class = None ) : <EOL> self . args = args <EOL> self . log = log <EOL> self . backend = ( backend_class or self . BackendClass ) ( <EOL> self . log , self . args . tarsnap_options , <EOL> dryrun = getattr ( self . args , '<STR_LIT>' , False ) ) <EOL> @ classmethod <EOL> def setup_arg_parser ( self , parser ) : <EOL> pass <EOL> @ classmethod <EOL> def validate_args ( self , args ) : <EOL> pass <EOL> def run ( self , job ) : <EOL> raise NotImplementedError ( ) <EOL> class ListCommand ( Command ) : <EOL> help = '<STR_LIT>' <EOL> description = '<STR_LIT>' <EOL> def run ( self , job ) : <EOL> backups = self . backend . get_backups ( job ) <EOL> self . log . info ( '<STR_LIT:%s>' % job . name ) <EOL> backups = [ ( name , time ) for name , time in backups . items ( ) ] <EOL> backups . sort ( cmp = lambda x , y : - cmp ( x [ <NUM_LIT:1> ] , y [ <NUM_LIT:1> ] ) ) <EOL> for backup , _ in backups : <EOL> print "<STR_LIT>" % backup <EOL> class ExpireCommand ( Command ) : <EOL> help = '<STR_LIT>' <EOL> description = '<STR_LIT>' '<STR_LIT>' <EOL> @ classmethod <EOL> def setup_arg_parser ( self , parser ) : <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' ) <EOL> def expire ( self , job ) : <EOL> if not job . deltas : <EOL> self . log . info ( ( "<STR_LIT>" ) % job . name ) <EOL> return <EOL> self . backend . expire ( job ) <EOL> def run ( self , job ) : <EOL> self . expire ( job ) <EOL> class MakeCommand ( ExpireCommand ) : <EOL> help = '<STR_LIT>' <EOL> description = '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' <EOL> @ classmethod <EOL> def setup_arg_parser ( self , parser ) : <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , <EOL> help = '<STR_LIT>' , ) <EOL> parser . add_argument ( '<STR_LIT>' , dest = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' , default = None , <EOL> help = '<STR_LIT>' ) <EOL> @ classmethod <EOL> def validate_args ( self , args ) : <EOL> if not args . config and not args . target : <EOL> raise ArgumentError ( '<STR_LIT>' '<STR_LIT>' ) <EOL> if not args . config and not args . deltas and not args . no_expire : <EOL> raise ArgumentError ( '<STR_LIT>' '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if not args . config and not args . sources : <EOL> raise ArgumentError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def run ( self , job ) : <EOL> if not job . sources : <EOL> self . log . info ( ( "<STR_LIT>" ) % job . name ) <EOL> return <EOL> if job . exec_before : <EOL> self . backend . _exec_util ( job . exec_before ) <EOL> sources_missing = False <EOL> if not job . force : <EOL> for source in job . sources : <EOL> if not path . exists ( source ) : <EOL> sources_missing = True <EOL> break <EOL> if path . isdir ( source ) and not os . listdir ( source ) : <EOL> sources_missing = True <EOL> break <EOL> skipped = False <EOL> if sources_missing : <EOL> if job . name : <EOL> self . log . info ( ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % job . name ) <EOL> else : <EOL> self . log . info ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> skipped = True <EOL> else : <EOL> try : <EOL> self . backend . make ( job ) <EOL> except Exception : <EOL> self . log . exception ( ( "<STR_LIT>" ) <EOL> % job . name ) <EOL> if job . exec_after : <EOL> self . backend . _exec_util ( job . exec_after ) <EOL> if not skipped and not self . args . no_expire : <EOL> self . expire ( job ) <EOL> COMMANDS = { <EOL> '<STR_LIT>' : MakeCommand , <EOL> '<STR_LIT>' : ExpireCommand , <EOL> '<STR_LIT:list>' : ListCommand , <EOL> } <EOL> PLUGINS = [ <EOL> ] <EOL> def parse_args ( argv ) : <EOL> """<STR_LIT>""" <EOL> parser = argparse . ArgumentParser ( <EOL> description = '<STR_LIT>' ) <EOL> group = parser . add_mutually_exclusive_group ( ) <EOL> group . add_argument ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , dest = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> group . add_argument ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , dest = '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> parser . add_argument ( '<STR_LIT>' , metavar = ( '<STR_LIT:name>' , '<STR_LIT:value>' ) , nargs = '<STR_LIT:+>' , <EOL> dest = '<STR_LIT>' , default = [ ] , action = '<STR_LIT>' , <EOL> help = '<STR_LIT>' , ) <EOL> parser . add_argument ( '<STR_LIT>' , '<STR_LIT:-c>' , help = '<STR_LIT>' ) <EOL> group = parser . add_argument_group ( <EOL> description = '<STR_LIT>' '<STR_LIT>' ) <EOL> group . add_argument ( '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> group . add_argument ( '<STR_LIT>' , nargs = '<STR_LIT:+>' , help = '<STR_LIT>' , <EOL> default = [ ] ) <EOL> group . add_argument ( '<STR_LIT>' , '<STR_LIT>' , metavar = '<STR_LIT>' , <EOL> type = timedelta_string , <EOL> help = '<STR_LIT>' , nargs = '<STR_LIT:+>' ) <EOL> group . add_argument ( '<STR_LIT>' , '<STR_LIT>' , help = '<STR_LIT>' ) <EOL> for plugin in PLUGINS : <EOL> plugin . setup_arg_parser ( parser ) <EOL> parser . add_argument ( '<STR_LIT:->' , dest = '<STR_LIT>' , action = "<STR_LIT:store_true>" , <EOL> help = argparse . SUPPRESS ) <EOL> subparsers = parser . add_subparsers ( <EOL> title = "<STR_LIT>" , description = "<STR_LIT>" ) <EOL> for cmd_name , cmd_klass in COMMANDS . iteritems ( ) : <EOL> subparser = subparsers . add_parser ( cmd_name , help = cmd_klass . help , <EOL> description = cmd_klass . description , <EOL> add_help = False ) <EOL> subparser . set_defaults ( command = cmd_klass ) <EOL> group = subparser . add_argument_group ( <EOL> title = "<STR_LIT>" ) <EOL> group . add_argument ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT>' , <EOL> default = argparse . SUPPRESS , <EOL> help = '<STR_LIT>' ) <EOL> cmd_klass . setup_arg_parser ( group ) <EOL> subparser . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , nargs = '<STR_LIT:*>' , <EOL> help = '<STR_LIT>' ) <EOL> group = parser . add_argument_group ( title = '<STR_LIT>' ) <EOL> group . add_argument ( <EOL> '<STR_LIT>' , metavar = '<STR_LIT>' , nargs = '<STR_LIT:*>' , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( argv ) <EOL> if args . config and ( args . target or args . dateformat or args . deltas or <EOL> args . sources ) : <EOL> raise ArgumentError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if args . jobs and not args . config : <EOL> raise ArgumentError ( ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % "<STR_LIT:U+002CU+0020>" . join ( args . jobs ) ) <EOL> args . command . validate_args ( args ) <EOL> return args <EOL> def main ( argv ) : <EOL> try : <EOL> args = parse_args ( argv ) <EOL> except ArgumentError , e : <EOL> print "<STR_LIT>" % e <EOL> return <NUM_LIT:1> <EOL> level = logging . WARNING if args . quiet else ( <EOL> logging . DEBUG if args . verbose else logging . INFO ) <EOL> ch = logging . StreamHandler ( ) <EOL> ch . setFormatter ( logging . Formatter ( "<STR_LIT>" ) ) <EOL> log = logging . getLogger ( ) <EOL> log . setLevel ( level ) <EOL> log . addHandler ( ch ) <EOL> if args . config : <EOL> try : <EOL> jobs , global_config = config . load_config_from_file ( args . config ) <EOL> except config . ConfigError , e : <EOL> log . fatal ( '<STR_LIT>' % e ) <EOL> return <NUM_LIT:1> <EOL> else : <EOL> jobs = { None : Job ( ** { '<STR_LIT:target>' : args . target , '<STR_LIT>' : args . dateformat , <EOL> '<STR_LIT>' : args . deltas , '<STR_LIT>' : args . sources } ) } <EOL> global_config = { } <EOL> if args . jobs : <EOL> unknown = set ( args . jobs ) - set ( jobs . keys ( ) ) <EOL> if unknown : <EOL> log . fatal ( '<STR_LIT>' % "<STR_LIT:U+002CU+0020>" . join ( unknown ) ) <EOL> return <NUM_LIT:1> <EOL> jobs_to_run = dict ( [ ( n , j ) for n , j in jobs . iteritems ( ) if n in args . jobs ] ) <EOL> else : <EOL> jobs_to_run = jobs <EOL> command = args . command ( args , log ) <EOL> try : <EOL> for job in jobs_to_run . values ( ) : <EOL> command . run ( job ) <EOL> for plugin in PLUGINS : <EOL> plugin . all_jobs_done ( args , global_config , args . command ) <EOL> except TarsnapError , e : <EOL> log . fatal ( "<STR_LIT>" % e ) <EOL> return <NUM_LIT:1> <EOL> def run ( ) : <EOL> sys . exit ( main ( sys . argv [ <NUM_LIT:1> : ] ) or <NUM_LIT:0> ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> run ( ) </s>
