<s> """<STR_LIT>""" <EOL> import math <EOL> import json <EOL> import pytz <EOL> import re <EOL> import logging <EOL> import os . path <EOL> import time <EOL> import sys <EOL> import traceback <EOL> import glaciercorecalls <EOL> import select <EOL> import hashlib <EOL> import fcntl <EOL> import termios <EOL> import struct <EOL> import boto <EOL> import boto . sdb <EOL> from boto import sns <EOL> from functools import wraps <EOL> from dateutil . parser import parse as dtparse <EOL> from datetime import datetime <EOL> from pprint import pformat <EOL> from glaciercorecalls import GlacierConnection , GlacierWriter <EOL> from glacierexception import * <EOL> class log_class_call ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , start , finish , getter = None ) : <EOL> """<STR_LIT>""" <EOL> self . start = start <EOL> self . finish = finish <EOL> self . getter = getter <EOL> def __call__ ( self , fn ) : <EOL> def wrapper ( * args , ** kwargs ) : <EOL> that = args [ <NUM_LIT:0> ] <EOL> that . logger . debug ( self . start ) <EOL> ret = fn ( * args , ** kwargs ) <EOL> that . logger . debug ( self . finish ) <EOL> if self . getter : <EOL> that . logger . debug ( pformat ( self . getter ( ret ) ) ) <EOL> else : <EOL> that . logger . debug ( pformat ( ret ) ) <EOL> return ret <EOL> wrapper . func_name = fn . func_name <EOL> if hasattr ( fn , '<STR_LIT>' ) : <EOL> wrapper . __name__ = self . name = fn . __name__ <EOL> if hasattr ( fn , '<STR_LIT>' ) : <EOL> wrapper . __doc__ = fn . __doc__ <EOL> if hasattr ( fn , '<STR_LIT>' ) : <EOL> wrapper . __module__ = fn . __module__ <EOL> return wrapper <EOL> class mmap ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , file ) : <EOL> self . file = file <EOL> self . size = os . fstat ( self . file . fileno ( ) ) . st_size <EOL> def __getitem__ ( self , key ) : <EOL> self . file . seek ( key . start ) <EOL> if key . stop is None : <EOL> stop = self . size <EOL> else : <EOL> stop = key . stop <EOL> return self . file . read ( stop - key . start ) <EOL> class GlacierWrapper ( object ) : <EOL> """<STR_LIT>""" <EOL> VAULT_NAME_ALLOWED_CHARACTERS = "<STR_LIT>" <EOL> ID_ALLOWED_CHARACTERS = "<STR_LIT>" <EOL> MAX_VAULT_NAME_LENGTH = <NUM_LIT:255> <EOL> MAX_VAULT_DESCRIPTION_LENGTH = <NUM_LIT> <EOL> MAX_PARTS = <NUM_LIT> <EOL> AVAILABLE_REGIONS = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> AVAILABLE_REGIONS_MESSAGE = """<STR_LIT>""" <EOL> def setuplogging ( self , logfile , loglevel , logtostdout ) : <EOL> """<STR_LIT>""" <EOL> levels = { '<STR_LIT:3>' : logging . CRITICAL , <EOL> '<STR_LIT>' : logging . CRITICAL , <EOL> '<STR_LIT:2>' : logging . ERROR , <EOL> '<STR_LIT>' : logging . ERROR , <EOL> '<STR_LIT:1>' : logging . WARNING , <EOL> '<STR_LIT>' : logging . WARNING , <EOL> '<STR_LIT:0>' : logging . INFO , <EOL> '<STR_LIT>' : logging . INFO , <EOL> '<STR_LIT>' : logging . DEBUG , <EOL> '<STR_LIT>' : logging . DEBUG } <EOL> loglevel = '<STR_LIT>' if not loglevel in levels . keys ( ) else levels [ loglevel ] <EOL> datefmt = '<STR_LIT>' <EOL> logformat = '<STR_LIT>' <EOL> if logtostdout : <EOL> logging . basicConfig ( level = loglevel , <EOL> stream = sys . stdout , <EOL> format = logformat , <EOL> datefmt = datefmt ) <EOL> elif logfile : <EOL> try : <EOL> open ( logfile , '<STR_LIT:a>' ) <EOL> except IOError : <EOL> logging . basicConfig ( level = loglevel , <EOL> stream = sys . stderr , <EOL> format = logformat , <EOL> datefmt = datefmt ) <EOL> else : <EOL> logging . basicConfig ( level = loglevel , <EOL> filename = logfile , <EOL> format = logformat , <EOL> datefmt = datefmt ) <EOL> else : <EOL> logging . basicConfig ( level = '<STR_LIT>' , <EOL> stream = sys . stdout , <EOL> format = logformat , <EOL> datefmt = datefmt ) <EOL> def glacier_connect ( func ) : <EOL> """<STR_LIT>""" <EOL> @ wraps ( func ) <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def glacier_connect_wrap ( * args , ** kwargs ) : <EOL> self = args [ <NUM_LIT:0> ] <EOL> if not hasattr ( self , "<STR_LIT>" ) or ( hasattr ( self , "<STR_LIT>" ) and not self . glacierconn ) : <EOL> try : <EOL> self . logger . debug ( """<STR_LIT>""" , <EOL> self . aws_access_key , <EOL> self . aws_secret_key , <EOL> self . region ) <EOL> self . glacierconn = GlacierConnection ( self . aws_access_key , <EOL> self . aws_secret_key , <EOL> region_name = self . region ) <EOL> except boto . exception . AWSConnectionError as e : <EOL> raise ConnectionException ( <EOL> "<STR_LIT>" , <EOL> cause = e . cause , <EOL> code = "<STR_LIT>" ) <EOL> return func ( * args , ** kwargs ) <EOL> return glacier_connect_wrap <EOL> def sdb_connect ( func ) : <EOL> """<STR_LIT>""" <EOL> @ wraps ( func ) <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def sdb_connect_wrap ( * args , ** kwargs ) : <EOL> self = args [ <NUM_LIT:0> ] <EOL> if not self . bookkeeping : <EOL> return func ( * args , ** kwargs ) <EOL> if not self . bookkeeping_domain_name : <EOL> raise InputException ( <EOL> '''<STR_LIT>''' , <EOL> code = "<STR_LIT>" ) <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> try : <EOL> self . logger . debug ( """<STR_LIT>""" , <EOL> self . bookkeeping_domain_name , <EOL> self . aws_access_key , <EOL> self . aws_secret_key ) <EOL> self . sdb_conn = boto . sdb . connect_to_region ( <EOL> self . sdb_region , <EOL> aws_access_key_id = self . sdb_access_key , <EOL> aws_secret_access_key = self . sdb_secret_key ) <EOL> domain_name = self . bookkeeping_domain_name <EOL> self . sdb_domain = self . sdb_conn . create_domain ( domain_name ) <EOL> except ( boto . exception . AWSConnectionError , boto . exception . SDBResponseError ) as e : <EOL> raise ConnectionException ( <EOL> "<STR_LIT>" , <EOL> cause = e , <EOL> code = "<STR_LIT>" ) <EOL> return func ( * args , ** kwargs ) <EOL> return sdb_connect_wrap <EOL> def sns_connect ( func ) : <EOL> """<STR_LIT>""" <EOL> @ wraps ( func ) <EOL> def sns_connect_wrap ( * args , ** kwargs ) : <EOL> self = args [ <NUM_LIT:0> ] <EOL> if not hasattr ( self , "<STR_LIT>" ) : <EOL> try : <EOL> self . sns_conn = boto . sns . connect_to_region ( <EOL> aws_access_key_id = self . aws_access_key , <EOL> aws_secret_access_key = self . aws_secret_key , <EOL> region_name = self . region ) <EOL> except boto . exception . AWSConnectionError as e : <EOL> raise ConnectionException ( <EOL> "<STR_LIT>" , <EOL> cause = e . cause , <EOL> code = "<STR_LIT>" ) <EOL> return func ( * args , ** kwargs ) <EOL> return sns_connect_wrap <EOL> @ log_class_call ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> def _check_vault_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> if len ( name ) > self . MAX_VAULT_NAME_LENGTH : <EOL> raise InputException ( <EOL> u"<STR_LIT>" % self . MAX_VAULT_NAME_LENGTH , <EOL> cause = "<STR_LIT>" % self . MAX_VAULT_NAME_LENGTH , <EOL> code = "<STR_LIT>" ) <EOL> if len ( name ) == <NUM_LIT:0> : <EOL> raise InputException ( <EOL> u"<STR_LIT>" , <EOL> cause = '<STR_LIT>' , <EOL> code = "<STR_LIT>" ) <EOL> m = re . match ( self . VAULT_NAME_ALLOWED_CHARACTERS , name ) <EOL> if ( m . end ( ) if m else <NUM_LIT:0> ) != len ( name ) : <EOL> raise InputException ( <EOL> u"""<STR_LIT>""" , <EOL> cause = '<STR_LIT>' , <EOL> code = "<STR_LIT>" ) <EOL> return True <EOL> @ log_class_call ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> def _check_vault_description ( self , description ) : <EOL> """<STR_LIT>""" <EOL> if len ( description ) > self . MAX_VAULT_DESCRIPTION_LENGTH : <EOL> raise InputException ( <EOL> u"<STR_LIT>" % self . MAX_VAULT_DESCRIPTION_LENGTH , <EOL> cause = '<STR_LIT>' % self . MAX_VAULT_DESCRIPTION_LENGTH , <EOL> code = "<STR_LIT>" ) <EOL> for char in description : <EOL> n = ord ( char ) <EOL> if n < <NUM_LIT:32> or n > <NUM_LIT> : <EOL> raise InputException ( <EOL> u"""<STR_LIT>""" , <EOL> cause = "<STR_LIT>" , <EOL> code = "<STR_LIT>" ) <EOL> return True <EOL> @ log_class_call ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> def _check_id ( self , amazon_id , id_type ) : <EOL> """<STR_LIT>""" <EOL> length = { '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> } <EOL> self . logger . debug ( '<STR_LIT>' % id_type ) <EOL> if len ( amazon_id ) != length [ id_type ] : <EOL> raise InputException ( <EOL> '<STR_LIT>' % ( id_type , length [ id_type ] , len ( amazon_id ) ) , <EOL> cause = '<STR_LIT>' % id_type , <EOL> code = "<STR_LIT>" ) <EOL> m = re . match ( self . ID_ALLOWED_CHARACTERS , amazon_id ) <EOL> if ( m . end ( ) if m else <NUM_LIT:0> ) != len ( amazon_id ) : <EOL> raise InputException ( u"""<STR_LIT>""" % id_type , <EOL> cause = '<STR_LIT>' % id_type , <EOL> code = "<STR_LIT>" ) <EOL> return True <EOL> @ log_class_call ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> def _check_region ( self , region ) : <EOL> """<STR_LIT>""" <EOL> if not region in self . AVAILABLE_REGIONS : <EOL> raise InputException ( <EOL> self . AVAILABLE_REGIONS_MESSAGE , <EOL> cause = '<STR_LIT>' % region , <EOL> code = '<STR_LIT>' ) <EOL> return True <EOL> def _check_part_size ( self , part_size , total_size ) : <EOL> """<STR_LIT>""" <EOL> def _part_size_for_total_size ( total_size ) : <EOL> return self . _next_power_of_2 ( <EOL> int ( math . ceil ( <EOL> float ( total_size ) / ( <NUM_LIT> * <NUM_LIT> * self . MAX_PARTS ) <EOL> ) ) ) <EOL> if part_size < <NUM_LIT:0> : <EOL> if total_size > <NUM_LIT:0> : <EOL> part_size = _part_size_for_total_size ( total_size ) <EOL> else : <EOL> part_size = GlacierWriter . DEFAULT_PART_SIZE <EOL> else : <EOL> ps = self . _next_power_of_2 ( part_size ) <EOL> if not ps == part_size : <EOL> self . logger . warning ( """<STR_LIT>""" % ( part_size , ps ) ) <EOL> part_size = ps <EOL> if total_size > part_size * <NUM_LIT> * <NUM_LIT> * self . MAX_PARTS : <EOL> part_size = _part_size_for_total_size ( total_size ) <EOL> self . logger . warning ( "<STR_LIT>" % part_size ) <EOL> return part_size <EOL> def _next_power_of_2 ( self , v ) : <EOL> """<STR_LIT>""" <EOL> if v == <NUM_LIT:0> : <EOL> return <NUM_LIT:1> <EOL> v -= <NUM_LIT:1> <EOL> v |= v >> <NUM_LIT:1> <EOL> v |= v >> <NUM_LIT:2> <EOL> v |= v >> <NUM_LIT:4> <EOL> v |= v >> <NUM_LIT:8> <EOL> v |= v >> <NUM_LIT:16> <EOL> return v + <NUM_LIT:1> <EOL> def _bold ( self , msg ) : <EOL> """<STR_LIT>""" <EOL> return u'<STR_LIT>' % msg <EOL> def _progress ( self , msg ) : <EOL> """<STR_LIT>""" <EOL> if sys . stdout . isatty ( ) : <EOL> cols = struct . unpack ( '<STR_LIT>' , fcntl . ioctl ( sys . stdout , termios . TIOCGWINSZ , '<STR_LIT>' ) ) [ <NUM_LIT:1> ] <EOL> msg = msg [ : cols ] if len ( msg ) > cols else msg <EOL> if len ( msg ) < cols : <EOL> for i in range ( cols - len ( msg ) ) : <EOL> msg += '<STR_LIT:U+0020>' <EOL> sys . stdout . write ( msg + '<STR_LIT:\r>' ) <EOL> sys . stdout . flush ( ) <EOL> def _size_fmt ( self , num , decimals = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> fmt = "<STR_LIT>" % decimals <EOL> for x in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if num < <NUM_LIT> : <EOL> return fmt % ( num , x ) <EOL> num /= <NUM_LIT> <EOL> return fmt % ( num , '<STR_LIT>' ) <EOL> def _decode_error_message ( self , e ) : <EOL> try : <EOL> e = json . loads ( e ) [ '<STR_LIT:message>' ] <EOL> except : <EOL> e = None <EOL> return e <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def lsvault ( self , limit = None ) : <EOL> """<STR_LIT>""" <EOL> marker = None <EOL> vault_list = [ ] <EOL> while True : <EOL> try : <EOL> response = self . glacierconn . list_vaults ( marker = marker ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> vault_list += response . copy ( ) [ '<STR_LIT>' ] <EOL> marker = response . copy ( ) [ '<STR_LIT>' ] <EOL> if limit and len ( vault_list ) >= limit : <EOL> vault_list = vault_list [ : limit ] <EOL> break <EOL> if not marker : <EOL> break <EOL> return vault_list <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def mkvault ( self , vault_name ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> try : <EOL> response = self . glacierconn . create_vault ( vault_name ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> return response . copy ( ) <EOL> @ glacier_connect <EOL> @ sdb_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def rmvault ( self , vault_name ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> try : <EOL> response = self . glacierconn . delete_vault ( vault_name ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> if self . bookkeeping : <EOL> query = "<STR_LIT>" % ( self . bookkeeping_domain_name , vault_name ) <EOL> result = self . sdb_domain . select ( query ) <EOL> try : <EOL> for item in result : <EOL> self . sdb_domain . delete_item ( item ) <EOL> self . logger . debug ( '<STR_LIT>' % item . name ) <EOL> except boto . exception . SDBResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> return response . copy ( ) <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def describevault ( self , vault_name ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> try : <EOL> response = self . glacierconn . describe_vault ( vault_name ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> return response . copy ( ) <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def list_jobs ( self , vault_name , completed = None , <EOL> status_code = None , limit = None ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> marker = None <EOL> job_list = [ ] <EOL> while True : <EOL> try : <EOL> response = self . glacierconn . list_jobs ( vault_name , <EOL> completed = completed , <EOL> status_code = status_code , <EOL> marker = marker ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> job_list += response . copy ( ) [ '<STR_LIT>' ] <EOL> marker = response . copy ( ) [ '<STR_LIT>' ] <EOL> if limit and len ( job_list ) >= limit : <EOL> job_list = job_list [ : limit ] <EOL> break <EOL> if not marker : <EOL> break <EOL> return job_list <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def describejob ( self , vault_name , job_id ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> self . _check_id ( job_id , '<STR_LIT>' ) <EOL> try : <EOL> response = self . glacierconn . describe_job ( vault_name , job_id ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % job_id , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> return response . copy ( ) <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def abortmultipart ( self , vault_name , upload_id ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> self . _check_id ( upload_id , "<STR_LIT>" ) <EOL> try : <EOL> response = self . glacierconn . abort_multipart_upload ( vault_name , upload_id ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % upload_id , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> return response . copy ( ) <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def listmultiparts ( self , vault_name , limit = None ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> marker = None <EOL> uploads = [ ] <EOL> while True : <EOL> try : <EOL> response = self . glacierconn . list_multipart_uploads ( vault_name , <EOL> marker = marker ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> uploads += response . copy ( ) [ '<STR_LIT>' ] <EOL> marker = response . copy ( ) [ '<STR_LIT>' ] <EOL> if limit and len ( uploads ) >= limit : <EOL> uploads = uploads [ : limit ] <EOL> break <EOL> if not marker : <EOL> break <EOL> return uploads <EOL> @ glacier_connect <EOL> @ sdb_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def upload ( self , vault_name , file_name , description , region , <EOL> stdin , alternative_name , part_size , uploadid , resume ) : <EOL> """<STR_LIT>""" <EOL> if self . logger . getEffectiveLevel ( ) == <NUM_LIT:10> : <EOL> logging . getLogger ( '<STR_LIT>' ) . setLevel ( logging . INFO ) <EOL> self . _check_vault_name ( vault_name ) <EOL> self . _check_region ( region ) <EOL> if not description : <EOL> description = file_name if file_name else '<STR_LIT>' <EOL> if description : <EOL> self . _check_vault_description ( description ) <EOL> if uploadid : <EOL> self . _check_id ( uploadid , '<STR_LIT>' ) <EOL> if resume and stdin : <EOL> raise InputException ( <EOL> '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> total_size = <NUM_LIT:0> <EOL> reader = None <EOL> mmapped_file = None <EOL> if not stdin : <EOL> if not file_name : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> code = '<STR_LIT>' ) <EOL> try : <EOL> f = open ( file_name , '<STR_LIT:rb>' ) <EOL> mmapped_file = mmap ( f ) <EOL> total_size = os . path . getsize ( file_name ) <EOL> except IOError as e : <EOL> raise InputException ( <EOL> "<STR_LIT>" % file_name , <EOL> cause = e , <EOL> code = '<STR_LIT>' ) <EOL> elif select . select ( [ sys . stdin , ] , [ ] , [ ] , <NUM_LIT:0.0> ) [ <NUM_LIT:0> ] : <EOL> reader = sys . stdin <EOL> total_size = <NUM_LIT:0> <EOL> else : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> code = '<STR_LIT>' ) <EOL> if uploadid : <EOL> self . logger . info ( '<STR_LIT>' % ( file_name if file_name else '<STR_LIT>' , vault_name ) ) <EOL> elif resume : <EOL> self . logger . info ( '<STR_LIT>' % ( file_name , vault_name ) ) <EOL> else : <EOL> self . logger . info ( '<STR_LIT>' % ( file_name if file_name else '<STR_LIT>' , vault_name , description ) ) <EOL> part_size = self . _check_part_size ( part_size , total_size ) <EOL> part_size_in_bytes = part_size * <NUM_LIT> * <NUM_LIT> <EOL> upload = None <EOL> if uploadid : <EOL> uploads = self . listmultiparts ( vault_name ) <EOL> for upload in uploads : <EOL> if uploadid == upload [ '<STR_LIT>' ] : <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> self . logger . debug ( upload ) <EOL> part_size_in_bytes = upload [ '<STR_LIT>' ] <EOL> break <EOL> else : <EOL> raise InputException ( <EOL> '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> writer = GlacierWriter ( self . glacierconn , vault_name , description = description , <EOL> part_size_in_bytes = part_size_in_bytes , uploadid = uploadid , logger = self . logger ) <EOL> if upload : <EOL> marker = None <EOL> while True : <EOL> try : <EOL> response = self . glacierconn . list_parts ( vault_name , uploadid , marker = marker ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % uploadid , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> list_parts_response = response . copy ( ) <EOL> current_position = <NUM_LIT:0> <EOL> stop = <NUM_LIT:0> <EOL> for part in list_parts_response [ '<STR_LIT>' ] : <EOL> start , stop = ( int ( p ) for p in part [ '<STR_LIT>' ] . split ( '<STR_LIT:->' ) ) <EOL> stop += <NUM_LIT:1> <EOL> if not start == current_position : <EOL> if stdin : <EOL> raise InputException ( <EOL> '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> if reader : <EOL> reader . seek ( start ) <EOL> if mmapped_file and stop > mmapped_file . size : <EOL> raise InputException ( <EOL> '<STR_LIT>' , <EOL> cause = '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> data = None <EOL> data = reader . read ( stop - start ) if reader else mmapped_file [ start : stop ] <EOL> if data : <EOL> data_hash = glaciercorecalls . tree_hash ( glaciercorecalls . chunk_hashes ( data ) ) <EOL> if glaciercorecalls . bytes_to_hex ( data_hash ) == part [ '<STR_LIT>' ] : <EOL> self . logger . debug ( '<STR_LIT>' % part [ '<STR_LIT>' ] ) <EOL> writer . tree_hashes . append ( data_hash ) <EOL> else : <EOL> raise InputException ( <EOL> '<STR_LIT>' , <EOL> cause = '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> else : <EOL> raise InputException ( <EOL> '<STR_LIT>' , <EOL> cause = '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> current_position += stop - start <EOL> marker = list_parts_response [ '<STR_LIT>' ] <EOL> writer . uploaded_size = stop <EOL> if not marker : <EOL> break <EOL> if total_size > <NUM_LIT:0> : <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) , <EOL> self . _size_fmt ( total_size ) , <EOL> self . _bold ( str ( int ( <NUM_LIT:100> * writer . uploaded_size / total_size ) ) ) ) <EOL> else : <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) ) <EOL> self . _progress ( msg ) <EOL> self . logger . info ( '<STR_LIT>' % self . _size_fmt ( stop ) ) <EOL> if total_size > <NUM_LIT:0> : <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) , <EOL> self . _size_fmt ( total_size ) , <EOL> self . _bold ( str ( int ( <NUM_LIT:100> * writer . uploaded_size / total_size ) ) ) ) <EOL> else : <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) ) <EOL> self . _progress ( msg ) <EOL> start_time = current_time = previous_time = time . time ( ) <EOL> start_bytes = writer . uploaded_size <EOL> while True : <EOL> if reader : <EOL> part = reader . read ( part_size_in_bytes ) <EOL> else : <EOL> if mmapped_file . size > writer . uploaded_size + part_size_in_bytes : <EOL> part = mmapped_file [ writer . uploaded_size : writer . uploaded_size + part_size_in_bytes ] <EOL> else : <EOL> part = mmapped_file [ writer . uploaded_size : ] <EOL> if not part : <EOL> break <EOL> writer . write ( part ) <EOL> current_time = time . time ( ) <EOL> overall_rate = int ( ( writer . uploaded_size - start_bytes ) / ( current_time - start_time ) ) <EOL> if total_size > <NUM_LIT:0> : <EOL> current_rate = int ( part_size_in_bytes / ( current_time - previous_time ) ) <EOL> if overall_rate > <NUM_LIT:0> : <EOL> time_left = ( total_size - writer . uploaded_size ) / overall_rate <EOL> eta_seconds = current_time + time_left <EOL> if datetime . fromtimestamp ( eta_seconds ) . day is not datetime . now ( ) . day : <EOL> eta_template = "<STR_LIT>" <EOL> else : <EOL> eta_template = "<STR_LIT>" <EOL> eta = time . strftime ( eta_template , <EOL> time . localtime ( eta_seconds ) ) <EOL> else : <EOL> time_left = "<STR_LIT>" <EOL> eta = "<STR_LIT>" <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) , <EOL> self . _size_fmt ( total_size ) , <EOL> self . _bold ( str ( int ( <NUM_LIT:100> * writer . uploaded_size / total_size ) ) ) , <EOL> self . _size_fmt ( current_rate , <NUM_LIT:2> ) , <EOL> self . _size_fmt ( overall_rate , <NUM_LIT:2> ) , <EOL> eta ) <EOL> else : <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) , <EOL> self . _size_fmt ( overall_rate , <NUM_LIT:2> ) ) <EOL> self . _progress ( msg ) <EOL> previous_time = current_time <EOL> self . logger . debug ( msg ) <EOL> writer . close ( ) <EOL> if not stdin : <EOL> f . close ( ) <EOL> current_time = time . time ( ) <EOL> overall_rate = int ( writer . uploaded_size / ( current_time - start_time ) ) <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( writer . uploaded_size ) , <EOL> self . _size_fmt ( overall_rate , <NUM_LIT:2> ) ) <EOL> self . _progress ( msg ) <EOL> self . logger . info ( msg ) <EOL> archive_id = writer . get_archive_id ( ) <EOL> sha256hash = writer . get_hash ( ) <EOL> location = writer . get_location ( ) <EOL> if self . bookkeeping : <EOL> self . logger . info ( '<STR_LIT>' ) <EOL> file_name = alternative_name if alternative_name else file_name <EOL> file_name = file_name if file_name else '<STR_LIT>' <EOL> file_attrs = { <EOL> '<STR_LIT>' : region , <EOL> '<STR_LIT>' : vault_name , <EOL> '<STR_LIT:filename>' : file_name , <EOL> '<STR_LIT>' : archive_id , <EOL> '<STR_LIT:location>' : location , <EOL> '<STR_LIT:description>' : description , <EOL> '<STR_LIT:date>' : '<STR_LIT:%s>' % datetime . utcnow ( ) . replace ( tzinfo = pytz . utc ) , <EOL> '<STR_LIT>' : sha256hash , <EOL> '<STR_LIT:size>' : writer . uploaded_size <EOL> } <EOL> self . sdb_domain . put_attributes ( file_attrs [ '<STR_LIT:filename>' ] , file_attrs ) <EOL> return ( archive_id , sha256hash ) <EOL> @ glacier_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def getarchive ( self , vault_name , archive_id ) : <EOL> """<STR_LIT>""" <EOL> results = None <EOL> self . _check_vault_name ( vault_name ) <EOL> self . _check_id ( archive_id , '<STR_LIT>' ) <EOL> job_list = self . list_jobs ( vault_name ) <EOL> for job in job_list : <EOL> if job [ '<STR_LIT>' ] == archive_id : <EOL> if job [ '<STR_LIT>' ] : <EOL> return ( '<STR_LIT>' , job , job [ '<STR_LIT>' ] ) <EOL> return ( '<STR_LIT>' , job , None ) <EOL> job_data = { '<STR_LIT>' : archive_id , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> try : <EOL> response = self . glacierconn . initiate_job ( vault_name , job_data ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % ( archive_id , vault_name ) , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> job = response . copy ( ) <EOL> return ( '<STR_LIT>' , job , None ) <EOL> @ glacier_connect <EOL> @ sdb_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def download ( self , vault_name , archive_id , part_size , <EOL> out_file_name = None , overwrite = False ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> self . _check_id ( archive_id , '<STR_LIT>' ) <EOL> job_list = self . list_jobs ( vault_name ) <EOL> job_id = None <EOL> for job in job_list : <EOL> if job [ '<STR_LIT>' ] == archive_id : <EOL> download_job = job <EOL> if not job [ '<STR_LIT>' ] : <EOL> raise CommunicationException ( <EOL> "<STR_LIT>" , <EOL> code = '<STR_LIT>' ) <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> break <EOL> else : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> code = '<STR_LIT>' ) <EOL> out_file = None <EOL> if out_file_name : <EOL> if os . path . isfile ( out_file_name ) and not overwrite : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> code = "<STR_LIT>" ) <EOL> try : <EOL> out_file = open ( out_file_name , '<STR_LIT:w>' ) <EOL> except IOError as e : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> cause = e , <EOL> code = '<STR_LIT>' ) <EOL> total_size = download_job [ '<STR_LIT>' ] <EOL> part_size_in_bytes = self . _check_part_size ( part_size , total_size ) * <NUM_LIT> * <NUM_LIT> <EOL> start_bytes = downloaded_size = <NUM_LIT:0> <EOL> hash_list = [ ] <EOL> start_time = current_time = previous_time = time . time ( ) <EOL> if out_file : <EOL> self . logger . debug ( '<STR_LIT>' % out_file_name ) <EOL> else : <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> while downloaded_size < total_size : <EOL> from_bytes = downloaded_size <EOL> to_bytes = min ( downloaded_size + part_size_in_bytes , total_size ) <EOL> try : <EOL> response = self . glacierconn . get_job_output ( vault_name , <EOL> download_job [ '<STR_LIT>' ] , <EOL> byte_range = ( from_bytes , to_bytes - <NUM_LIT:1> ) ) <EOL> data = response . read ( ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % archive_id , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> hash_list . append ( glaciercorecalls . chunk_hashes ( data ) [ <NUM_LIT:0> ] ) <EOL> downloaded_size = to_bytes <EOL> if out_file : <EOL> try : <EOL> out_file . write ( response . read ( ) ) <EOL> except IOError as e : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> cause = e , <EOL> code = '<STR_LIT>' ) <EOL> else : <EOL> sys . stdout . write ( response . read ( ) ) <EOL> sys . stdout . flush ( ) <EOL> current_time = time . time ( ) <EOL> overall_rate = int ( ( downloaded_size - start_bytes ) / ( current_time - start_time ) ) <EOL> current_rate = int ( part_size_in_bytes / ( current_time - previous_time ) ) <EOL> time_left = ( total_size - downloaded_size ) / overall_rate <EOL> eta_seconds = current_time + time_left <EOL> if datetime . fromtimestamp ( eta_seconds ) . day is not datetime . now ( ) . day : <EOL> eta_template = "<STR_LIT>" <EOL> else : <EOL> eta_template = "<STR_LIT>" <EOL> eta = time . strftime ( eta_template , time . localtime ( eta_seconds ) ) <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( downloaded_size ) , <EOL> self . _size_fmt ( total_size ) , <EOL> self . _bold ( str ( int ( <NUM_LIT:100> * downloaded_size / total_size ) ) ) , <EOL> self . _size_fmt ( current_rate , <NUM_LIT:2> ) , <EOL> self . _size_fmt ( overall_rate , <NUM_LIT:2> ) , <EOL> eta ) <EOL> self . _progress ( msg ) <EOL> previous_time = current_time <EOL> self . logger . debug ( msg ) <EOL> if out_file : <EOL> out_file . close ( ) <EOL> if glaciercorecalls . bytes_to_hex ( glaciercorecalls . tree_hash ( hash_list ) ) != download_job [ '<STR_LIT>' ] : <EOL> raise CommunicationException ( <EOL> "<STR_LIT>" , <EOL> code = "<STR_LIT>" , <EOL> cause = None ) <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> current_time = time . time ( ) <EOL> overall_rate = int ( downloaded_size / ( current_time - start_time ) ) <EOL> msg = '<STR_LIT>' % ( self . _size_fmt ( downloaded_size ) , <EOL> self . _size_fmt ( overall_rate , <NUM_LIT:2> ) ) <EOL> self . _progress ( msg ) <EOL> self . logger . info ( msg ) <EOL> @ glacier_connect <EOL> @ sdb_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def search ( self , vault = None , region = None , file_name = None , search_term = None ) : <EOL> """<STR_LIT>""" <EOL> if not self . bookkeeping : <EOL> raise InputException ( <EOL> "<STR_LIT>" , <EOL> cause = '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> if vault : <EOL> self . _check_vault_name ( vault ) <EOL> if region : <EOL> self . _check_region ( region ) <EOL> self . logger . debug ( '<STR_LIT>' % <EOL> ( vault , region , file_name , search_term ) ) <EOL> search_params = [ ] <EOL> if region : <EOL> search_params += [ "<STR_LIT>" % ( region , ) ] <EOL> if vault : <EOL> search_params += [ "<STR_LIT>" % ( vault , ) ] <EOL> if file_name : <EOL> search_params += [ "<STR_LIT>" + file_name . replace ( "<STR_LIT:'>" , "<STR_LIT>" ) + "<STR_LIT>" ] <EOL> if search_term : <EOL> search_params += [ "<STR_LIT>" + search_term . replace ( "<STR_LIT:'>" , "<STR_LIT>" ) + "<STR_LIT>" ] <EOL> if search_params : <EOL> search_params = "<STR_LIT>" . join ( search_params ) <EOL> query = '<STR_LIT>' % ( self . bookkeeping_domain_name , search_params ) <EOL> else : <EOL> query = '<STR_LIT>' % self . bookkeeping_domain_name <EOL> self . logger . debug ( '<STR_LIT>' % query ) <EOL> result = self . sdb_domain . select ( query ) <EOL> items = [ ] <EOL> try : <EOL> for item in result : <EOL> self . logger . debug ( '<STR_LIT>' % item ) <EOL> if item . has_key ( '<STR_LIT>' ) : <EOL> items . append ( item ) <EOL> except boto . exception . SDBResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % search_params , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> return items <EOL> @ glacier_connect <EOL> @ sdb_connect <EOL> @ log_class_call ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> def rmarchive ( self , vault_name , archive_id ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> self . _check_id ( archive_id , '<STR_LIT>' ) <EOL> try : <EOL> self . glacierconn . delete_archive ( vault_name , archive_id ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % ( archive_id , vault_name ) , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> if self . bookkeeping : <EOL> try : <EOL> item = self . sdb_domain . get_item ( archive_id ) <EOL> if item : <EOL> self . sdb_domain . delete_item ( item ) <EOL> except boto . exception . SDBResponseError as e : <EOL> raise CommunicationException ( <EOL> "<STR_LIT>" , <EOL> code = "<STR_LIT>" , <EOL> cause = e ) <EOL> @ glacier_connect <EOL> @ sdb_connect <EOL> @ log_class_call ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> def inventory ( self , vault_name , refresh ) : <EOL> """<STR_LIT>""" <EOL> self . _check_vault_name ( vault_name ) <EOL> inventory = None <EOL> inventory_job = None <EOL> if not refresh : <EOL> job_list = self . list_jobs ( vault_name ) <EOL> inventory_done = False <EOL> for job in job_list : <EOL> if job [ '<STR_LIT>' ] == "<STR_LIT>" : <EOL> if job [ '<STR_LIT>' ] : <EOL> self . logger . debug ( '<STR_LIT>' % job ) <EOL> d = dtparse ( job [ '<STR_LIT>' ] ) . replace ( tzinfo = pytz . utc ) <EOL> job [ '<STR_LIT>' ] = d <EOL> inventory_done = True <EOL> inventory_job = job <EOL> break <EOL> self . logger . debug ( '<STR_LIT>' % job ) <EOL> inventory_job = job <EOL> if inventory_done : <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> response = self . glacierconn . get_job_output ( vault_name , inventory_job [ '<STR_LIT>' ] ) <EOL> inventory = response . copy ( ) <EOL> archives = [ ] <EOL> if self . bookkeeping and len ( inventory [ '<STR_LIT>' ] ) > <NUM_LIT:0> : <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> items = { } <EOL> for item in inventory [ '<STR_LIT>' ] : <EOL> items [ item [ '<STR_LIT>' ] ] = { <EOL> '<STR_LIT>' : vault_name , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT:description>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT:date>' : '<STR_LIT:%s>' % dtparse ( item [ '<STR_LIT>' ] ) . replace ( tzinfo = pytz . utc ) , <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT:size>' : item [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : self . region <EOL> } <EOL> archives . append ( item [ '<STR_LIT>' ] ) <EOL> if len ( items ) == <NUM_LIT> : <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> try : <EOL> self . sdb_domain . batch_put_attributes ( items ) <EOL> except boto . exception . SDBResponseError as e : <EOL> raise CommunicationException ( <EOL> "<STR_LIT>" , <EOL> cause = e , <EOL> code = "<STR_LIT>" ) <EOL> items = { } <EOL> if items : <EOL> self . logger . debug ( '<STR_LIT>' % len ( items ) ) <EOL> try : <EOL> self . sdb_domain . batch_put_attributes ( items ) <EOL> except boto . exception . SDBResponseError as e : <EOL> raise CommunicationException ( <EOL> "<STR_LIT>" , <EOL> cause = e , <EOL> code = "<STR_LIT>" ) <EOL> query = "<STR_LIT>" % ( self . bookkeeping_domain_name , vault_name ) <EOL> result = self . sdb_domain . select ( query ) <EOL> try : <EOL> for item in result : <EOL> if not item . name in archives : <EOL> self . sdb_domain . delete_item ( item ) <EOL> self . logger . debug ( '<STR_LIT>' % item . name ) <EOL> except boto . exception . SDBResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> if refresh or not inventory_job : <EOL> self . logger . debug ( '<STR_LIT>' ) <EOL> job_data = { '<STR_LIT>' : '<STR_LIT>' } <EOL> try : <EOL> new_job = self . glacierconn . initiate_job ( vault_name , job_data ) <EOL> except boto . glacier . exceptions . UnexpectedHTTPResponseError as e : <EOL> raise ResponseException ( <EOL> '<STR_LIT>' % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> inventory_job = self . describejob ( vault_name , new_job [ '<STR_LIT>' ] ) <EOL> return ( inventory_job , inventory ) <EOL> def get_tree_hash ( self , file_name ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> reader = open ( file_name , '<STR_LIT:rb>' ) <EOL> except IOError as e : <EOL> raise InputException ( <EOL> "<STR_LIT>" % file_name , <EOL> cause = e , <EOL> code = '<STR_LIT>' ) <EOL> hashes = [ hashlib . sha256 ( part ) . digest ( ) for part in iter ( ( lambda : reader . read ( <NUM_LIT> * <NUM_LIT> ) ) , '<STR_LIT>' ) ] <EOL> return glaciercorecalls . bytes_to_hex ( glaciercorecalls . tree_hash ( hashes ) ) <EOL> def _init_events_for_vault ( self , vault_name , topic ) : <EOL> config = { <EOL> '<STR_LIT>' : topic , <EOL> '<STR_LIT>' : [ '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> } <EOL> return self . glacierconn . set_vault_notifications ( <EOL> vault_name = vault_name , <EOL> notification_config = config ) <EOL> def _init_events_for_vaults ( self , vaults , topic ) : <EOL> results = [ ] <EOL> for vault_name in vaults : <EOL> try : <EOL> import collections <EOL> result = collections . OrderedDict ( ) <EOL> except AttributeError : <EOL> result = dict ( ) <EOL> result [ "<STR_LIT>" ] = vault_name <EOL> result [ "<STR_LIT>" ] = self . _init_events_for_vault ( vault_name , topic ) [ u'<STR_LIT>' ] <EOL> results += [ result ] <EOL> return results <EOL> @ glacier_connect <EOL> @ sns_connect <EOL> def sns_sync ( self , sns_options , output ) : <EOL> options = sns_options <EOL> if not options [ '<STR_LIT>' ] : <EOL> topic = self . sns_conn . create_topic ( options [ '<STR_LIT>' ] ) [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if options . get ( '<STR_LIT>' , None ) : <EOL> vaults = options [ '<STR_LIT>' ] . split ( "<STR_LIT:U+002C>" ) <EOL> else : <EOL> vaults = [ fvault [ u'<STR_LIT>' ] . split ( "<STR_LIT>" ) [ - <NUM_LIT:1> ] for fvault in self . lsvault ( ) ] <EOL> return self . _init_events_for_vaults ( vaults , topic ) <EOL> else : <EOL> results = [ ] <EOL> for topic in options [ "<STR_LIT>" ] : <EOL> vaults = [ ] <EOL> if topic . get ( "<STR_LIT>" , None ) : <EOL> if topic . get ( "<STR_LIT>" ) . get ( '<STR_LIT>' , None ) : <EOL> vaults = topic . get ( "<STR_LIT>" ) . get ( '<STR_LIT>' ) . split ( "<STR_LIT:U+002C>" ) <EOL> vaults = vaults [ : - <NUM_LIT:1> ] if vaults [ - <NUM_LIT:1> ] == "<STR_LIT>" else vaults <EOL> methods = [ ] <EOL> if topic . get ( '<STR_LIT>' , None ) : <EOL> if topic . get ( '<STR_LIT>' ) . get ( '<STR_LIT>' , None ) : <EOL> methods = topic . get ( '<STR_LIT>' ) . get ( '<STR_LIT>' ) . split ( "<STR_LIT:;>" ) <EOL> methods = methods [ : - <NUM_LIT:1> ] if methods [ - <NUM_LIT:1> ] == "<STR_LIT>" else methods <EOL> if not vaults : <EOL> vaults = [ fvault [ u'<STR_LIT>' ] . split ( "<STR_LIT>" ) [ - <NUM_LIT:1> ] for fvault in self . lsvault ( ) ] <EOL> topic_arn = self . sns_conn . create_topic ( topic [ '<STR_LIT>' ] ) [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> topic_results = self . _init_events_for_vaults ( vaults , topic_arn ) <EOL> if methods : <EOL> for method in methods : <EOL> try : <EOL> protocol , endpoint = method . split ( '<STR_LIT:U+002C>' ) <EOL> except ValueError : <EOL> raise InputException ( <EOL> ( "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> cause = '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> results_subscribe = self . sns_subscribe ( protocol , endpoint , topic = topic_arn . split ( "<STR_LIT::>" ) [ - <NUM_LIT:1> ] , sns_options = sns_options ) <EOL> for i , vault in enumerate ( topic_results ) : <EOL> try : <EOL> import collections <EOL> result = collections . OrderedDict ( ) <EOL> except AttributeError : <EOL> result = dict ( ) <EOL> if output == "<STR_LIT>" : <EOL> if not i : <EOL> result [ '<STR_LIT>' ] = topic [ '<STR_LIT>' ] <EOL> if methods : <EOL> result [ "<STR_LIT>" ] = results_subscribe [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> else : <EOL> result [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> else : <EOL> result [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> result [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> else : <EOL> result [ '<STR_LIT>' ] = topic [ '<STR_LIT>' ] <EOL> if methods : <EOL> result [ "<STR_LIT>" ] = results_subscribe [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> else : <EOL> result [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> result [ "<STR_LIT>" ] = vault [ '<STR_LIT>' ] <EOL> result [ "<STR_LIT>" ] = vault [ '<STR_LIT>' ] <EOL> results += [ result ] <EOL> return results <EOL> @ glacier_connect <EOL> @ sns_connect <EOL> def sns_subscribe ( self , protocol , endpoint , topic , sns_options , vault_names = None ) : <EOL> all_topics = self . sns_conn . get_all_topics ( ) [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> topic_arn = self . sns_conn . create_topic ( topic ) [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if vault_names : <EOL> vaults = vault_names . split ( "<STR_LIT:U+002C>" ) <EOL> self . _init_events_for_vaults ( vaults , topic_arn ) <EOL> topic_arns = [ topic_arn ] <EOL> if len ( topic_arns ) : <EOL> try : <EOL> results = [ ] <EOL> for arn in topic_arns : <EOL> result = self . sns_conn . subscribe ( arn , protocol , endpoint ) [ '<STR_LIT>' ] <EOL> results += [ { '<STR_LIT>' : result [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : result [ '<STR_LIT>' ] [ '<STR_LIT>' ] } ] <EOL> return results <EOL> except boto . exception . BotoServerError as e : <EOL> raise ResponseException ( "<STR_LIT>" % vault_name , <EOL> cause = self . _decode_error_message ( e . body ) , <EOL> code = e . code ) <EOL> else : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> @ glacier_connect <EOL> @ sns_connect <EOL> def sns_list_topics ( self , sns_options ) : <EOL> topics = self . sns_conn . get_all_topics ( ) [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> results = [ ] <EOL> for topic in topics : <EOL> results += [ { "<STR_LIT>" : topic [ '<STR_LIT>' ] . split ( "<STR_LIT::>" ) [ - <NUM_LIT:1> ] , "<STR_LIT>" : topic [ '<STR_LIT>' ] } ] <EOL> return results <EOL> @ glacier_connect <EOL> @ sns_connect <EOL> def sns_list_subscriptions ( self , protocol , endpoint , topic , sns_options ) : <EOL> subscriptions = self . sns_conn . get_all_subscriptions ( ) [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> results = [ ] <EOL> for sub in subscriptions : <EOL> try : <EOL> import collections <EOL> result = collections . OrderedDict ( ) <EOL> except AttributeError : <EOL> result = dict ( ) <EOL> subtopic = sub [ '<STR_LIT>' ] . split ( "<STR_LIT::>" ) [ - <NUM_LIT:1> ] <EOL> subprotocol = sub [ '<STR_LIT>' ] <EOL> subendpoint = sub [ '<STR_LIT>' ] <EOL> subarn = sub [ '<STR_LIT>' ] <EOL> if topic and subtopic != topic : <EOL> continue <EOL> if protocol and subprotocol != protocol : <EOL> continue <EOL> if endpoint and subendpoint != endpoint : <EOL> continue <EOL> result [ '<STR_LIT>' ] = sub [ '<STR_LIT>' ] <EOL> result [ '<STR_LIT>' ] = subtopic <EOL> result [ '<STR_LIT>' ] = subprotocol <EOL> result [ '<STR_LIT>' ] = subendpoint <EOL> result [ '<STR_LIT>' ] = subarn <EOL> results += [ result ] <EOL> return results <EOL> @ glacier_connect <EOL> @ sns_connect <EOL> def sns_unsubscribe ( self , protocol , endpoint , topic , sns_options ) : <EOL> if not protocol or not endpoint or not topic : <EOL> raise InputException ( <EOL> ( "<STR_LIT>" <EOL> "<STR_LIT>" ) , <EOL> cause = '<STR_LIT>' , <EOL> code = '<STR_LIT>' ) <EOL> matching_subs = self . sns_list_subscriptions ( protocol , <EOL> endpoint , <EOL> topic , <EOL> sns_options = sns_options ) <EOL> unsubscribed = [ ] <EOL> for res in matching_subs : <EOL> if res [ '<STR_LIT>' ] != u'<STR_LIT>' : <EOL> self . sns_conn . unsubscribe ( res [ '<STR_LIT>' ] ) <EOL> unsubscribed += [ res ] <EOL> return unsubscribed <EOL> def __init__ ( self , aws_access_key , aws_secret_key , region , <EOL> bookkeeping = False , no_bookkeeping = None , bookkeeping_domain_name = None , <EOL> sdb_access_key = None , sdb_secret_key = None , sdb_region = None , <EOL> logfile = None , loglevel = '<STR_LIT>' , logtostdout = True ) : <EOL> """<STR_LIT>""" <EOL> self . aws_access_key = aws_access_key <EOL> self . aws_secret_key = aws_secret_key <EOL> self . bookkeeping = bookkeeping <EOL> if no_bookkeeping : <EOL> self . bookkeeping = False <EOL> self . bookkeeping_domain_name = bookkeeping_domain_name <EOL> self . region = region <EOL> self . sdb_access_key = sdb_access_key if sdb_access_key else aws_access_key <EOL> self . sdb_secret_key = sdb_secret_key if sdb_secret_key else aws_secret_key <EOL> self . sdb_region = sdb_region if sdb_region else region <EOL> self . setuplogging ( logfile , loglevel , logtostdout ) <EOL> self . logger = logging . getLogger ( self . __class__ . __name__ ) <EOL> self . _check_region ( region ) <EOL> self . logger . debug ( """<STR_LIT>""" , <EOL> aws_access_key , aws_secret_key , bookkeeping , <EOL> no_bookkeeping , <EOL> bookkeeping_domain_name , region , <EOL> sdb_access_key , sdb_secret_key , sdb_region , <EOL> logfile , loglevel , logtostdout ) </s>
