<s> """<STR_LIT>""" <EOL> import sys <EOL> import os <EOL> import pkgutil <EOL> import tarfile <EOL> from importlib import import_module <EOL> from inspect import getmembers , isclass , getmro <EOL> from tempfile import NamedTemporaryFile <EOL> from boto . s3 . connection import S3Connection , OrdinaryCallingFormat <EOL> from boto . s3 . key import Key <EOL> from . utils import now_field <EOL> from . config import get_mapping <EOL> from . data import Storable <EOL> from . simple import DBObject , Field <EOL> def is_backup_class ( cls ) : <EOL> """<STR_LIT>""" <EOL> return True if ( <EOL> isclass ( cls ) and <EOL> issubclass ( cls , Storable ) and <EOL> get_mapping ( cls , no_mapping_ok = True ) <EOL> ) else False <EOL> def backup_name ( cls ) : <EOL> """<STR_LIT>""" <EOL> return cls . __name__ + '<STR_LIT>' + cls . get_table_name ( ) <EOL> if sys . version_info >= ( <NUM_LIT:3> , <NUM_LIT:0> ) : <EOL> def write_line ( file_obj , line ) : <EOL> file_obj . write ( bytes ( str ( line ) + '<STR_LIT:\n>' , '<STR_LIT>' ) ) <EOL> def strip_line ( line ) : <EOL> return str ( line , '<STR_LIT>' ) . strip ( ) <EOL> else : <EOL> def write_line ( file_obj , line ) : <EOL> file_obj . write ( str ( line ) ) <EOL> file_obj . write ( '<STR_LIT:\n>' ) <EOL> def strip_line ( line ) : <EOL> return str ( line ) . strip ( ) <EOL> write_line . __doc__ = """<STR_LIT>""" <EOL> strip_line . __doc__ = """<STR_LIT>""" <EOL> class Backup ( object ) : <EOL> """<STR_LIT>""" <EOL> name = Field ( '<STR_LIT>' ) <EOL> timestamp = Field ( now_field ) <EOL> aws_access_key = Field ( '<STR_LIT>' ) <EOL> aws_secret_key = Field ( '<STR_LIT>' ) <EOL> bucketname = Field ( '<STR_LIT>' ) <EOL> backup_log = Field ( list ) <EOL> def setup ( self , * args , ** kwrds ) : <EOL> self . classes = dict ( ) <EOL> if not self . aws_access_key : <EOL> self . aws_access_key = os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if not self . aws_secret_key : <EOL> self . aws_secret_key = os . environ . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def add_package ( <EOL> self , <EOL> pkg_name , <EOL> recurse = True , <EOL> include_bases = True , <EOL> parent_pkg = None <EOL> ) : <EOL> """<STR_LIT>""" <EOL> if parent_pkg : <EOL> pkg = import_module ( '<STR_LIT:.>' + pkg_name , parent_pkg ) <EOL> else : <EOL> pkg = import_module ( pkg_name ) <EOL> for module_loader , name , ispkg in pkgutil . walk_packages ( pkg . __path__ ) : <EOL> if not ispkg : <EOL> mod = import_module ( '<STR_LIT:.>' + name , pkg_name ) <EOL> for name , member in getmembers ( mod ) : <EOL> if is_backup_class ( member ) : <EOL> self . add_class ( member , include_bases = include_bases ) <EOL> elif recurse : <EOL> self . add_package ( <EOL> pkg_name + '<STR_LIT:.>' + name , <EOL> recurse = True , <EOL> include_bases = include_bases , <EOL> parent_pkg = parent_pkg <EOL> ) <EOL> def add_class ( self , cls , include_bases = True ) : <EOL> """<STR_LIT>""" <EOL> if not is_backup_class ( cls ) : <EOL> return <NUM_LIT:0> <EOL> added = <NUM_LIT:0> <EOL> cls_name = backup_name ( cls ) <EOL> if cls_name not in self . classes : <EOL> self . classes [ cls_name ] = cls <EOL> self . log ( "<STR_LIT>" , cls_name ) <EOL> added = <NUM_LIT:1> <EOL> if include_bases : <EOL> for candidate_cls in getmro ( cls ) : <EOL> if is_backup_class ( cls ) : <EOL> added += self . add_class ( candidate_cls , include_bases = False ) <EOL> return added <EOL> def log ( self , entry , * args ) : <EOL> """<STR_LIT>""" <EOL> if args : <EOL> entry = entry % args <EOL> self . backup_log . append ( entry ) <EOL> def run_backup ( self ) : <EOL> """<STR_LIT>""" <EOL> self . log ( "<STR_LIT>" , now_field ( ) ) <EOL> self . log ( "<STR_LIT>" , self . timestamp ) <EOL> for fld in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> val = getattr ( self , fld , None ) <EOL> if not val : <EOL> self . log ( "<STR_LIT>" , fld ) <EOL> raise ValueError ( self . backup_log [ - <NUM_LIT:1> ] ) <EOL> backup_file = NamedTemporaryFile ( suffix = "<STR_LIT>" ) <EOL> backup_tarfile = tarfile . open ( fileobj = backup_file , mode = '<STR_LIT>' ) <EOL> for cls_name , cls in self . classes . items ( ) : <EOL> self . log ( "<STR_LIT>" , cls_name ) <EOL> rec_count = <NUM_LIT:0> <EOL> with NamedTemporaryFile ( ) as record_file : <EOL> for rec in cls . find_all ( ) : <EOL> write_line ( record_file , rec . to_data ( ) ) <EOL> rec_count += <NUM_LIT:1> <EOL> record_file . flush ( ) <EOL> backup_tarfile . add ( record_file . name , arcname = cls_name + '<STR_LIT>' ) <EOL> self . log ( "<STR_LIT>" , cls_name , rec_count ) <EOL> backup_tarfile . close ( ) <EOL> backup_file . flush ( ) <EOL> backup_size = os . stat ( backup_file . name ) [ <NUM_LIT:6> ] <EOL> key_name = ( '<STR_LIT>' + now_field ( ) + '<STR_LIT>' ) . replace ( '<STR_LIT::>' , '<STR_LIT:_>' ) <EOL> if os . environ . get ( '<STR_LIT>' , False ) or os . environ . get ( '<STR_LIT>' , False ) : <EOL> conn = S3Connection ( <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> is_secure = False , port = <NUM_LIT> , host = '<STR_LIT:localhost>' , <EOL> calling_format = OrdinaryCallingFormat ( ) <EOL> ) <EOL> else : <EOL> conn = S3Connection ( self . aws_access_key , self . aws_secret_key ) <EOL> bucket = conn . get_bucket ( self . bucketname ) <EOL> key = Key ( bucket ) <EOL> key . key = key_name <EOL> self . log ( <EOL> "<STR_LIT>" , <EOL> backup_file . name , <EOL> backup_size , <EOL> key_name <EOL> ) <EOL> key . set_contents_from_filename ( backup_file . name ) <EOL> self . log ( "<STR_LIT>" , backup_file . name ) <EOL> backup_file . close ( ) <EOL> self . log ( "<STR_LIT>" ) <EOL> return self . bucketname , key_name <EOL> Backup = DBObject ( table_name = '<STR_LIT>' ) ( Backup ) </s>
