def should_use_s3_to_create_replica(self):
        """ determine whether it makes sense to use S3 and not pg_basebackup """

        threshold_megabytes = self.wal_e.threshold_mb
        threshold_percent = self.wal_e.threshold_pct

        try:
            cmd = self.wal_e.cmd + ['backup-list', '--detail', 'LATEST']

            logger.debug('calling %r', cmd)
            wale_output = subprocess.check_output(cmd)

            reader = csv.DictReader(wale_output.decode('utf-8').splitlines(),
                                    dialect='excel-tab')
            rows = list(reader)
            if not len(rows):
                logger.warning('wal-e did not find any backups')
                return False

            # This check might not add much, it was performed in the previous
            # version of this code. since the old version rolled CSV parsing the
            # check may have been part of the CSV parsing.
            if len(rows) > 1:
                logger.warning(
                    'wal-e returned more than one row of backups: %r',
                    rows)
                return False

            backup_info = rows[0]
        except subprocess.CalledProcessError:
            logger.exception("could not query wal-e latest backup")
            return None

        try:
            backup_size = int(backup_info['expanded_size_bytes'])
            backup_start_segment = backup_info['wal_segment_backup_start']
            backup_start_offset = backup_info['wal_segment_offset_backup_start']
        except KeyError:
            logger.exception("unable to get some of WALE backup parameters")
            return None

        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
        # that we have to convert to hex and 'prepend' to the high offset digits.

        lsn_segment = backup_start_segment[8:16]
        # first 2 characters of the result are 0x and the last one is L
        lsn_offset = hex((int(backup_start_segment[16:32], 16) << 24) + int(backup_start_offset))[2:-1]

        # construct the LSN from the segment and offset
        backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)

        diff_in_bytes = backup_size
        attempts_no = 0
        while True:
            if self.master_connection:
                try:
                    # get the difference in bytes between the current WAL location and the backup start offset
                    with psycopg2.connect(self.master_connection) as con:
                        if con.server_version >= 100000:
                            wal_name = 'wal'
                            lsn_name = 'lsn'
                        else:
                            wal_name = 'xlog'
                            lsn_name = 'location'
                        con.autocommit = True
                        with con.cursor() as cur:
                            cur.execute(("SELECT CASE WHEN pg_catalog.pg_is_in_recovery()"
                                         " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE("
                                         "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, "
                                         "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)"
                                         " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint"
                                         " END").format(wal_name, lsn_name),
                                        (backup_start_lsn, backup_start_lsn, backup_start_lsn))

                            diff_in_bytes = int(cur.fetchone()[0])
                except psycopg2.Error:
                    logger.exception('could not determine difference with the master location')
                    if attempts_no < self.retries:  # retry in case of a temporarily connection issue
                        attempts_no = attempts_no + 1
                        time.sleep(RETRY_SLEEP_INTERVAL)
                        continue
                    else:
                        if not self.no_master:
                            return False  # do no more retries on the outer level
                        logger.info("continue with base backup from S3 since master is not available")
                        diff_in_bytes = 0
                        break
            else:
                # always try to use WAL-E if master connection string is not available
                diff_in_bytes = 0
            break

        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
        is_size_thresh_ok = diff_in_bytes < int(threshold_megabytes) * 1048576
        threshold_pct_bytes = backup_size * threshold_percent / 100.0
        is_percentage_thresh_ok = float(diff_in_bytes) < int(threshold_pct_bytes)
        are_thresholds_ok = is_size_thresh_ok and is_percentage_thresh_ok

        class Size(object):
            def __init__(self, n_bytes, prefix=None):
                self.n_bytes = n_bytes
                self.prefix = prefix

            def __repr__(self):
                if self.prefix is not None:
                    n_bytes = size_as_bytes(self.n_bytes, self.prefix)
                else:
                    n_bytes = self.n_bytes
                return repr_size(n_bytes)

        class HumanContext(object):
            def __init__(self, items):
                self.items = items

            def __repr__(self):
                return ', '.join('{}={!r}'.format(key, value)
                                 for key, value in self.items)

        human_context = repr(HumanContext([
            ('threshold_size', Size(threshold_megabytes, 'M')),
            ('threshold_percent', threshold_percent),
            ('threshold_percent_size', Size(threshold_pct_bytes)),
            ('backup_size', Size(backup_size)),
            ('backup_diff', Size(diff_in_bytes)),
            ('is_size_thresh_ok', is_size_thresh_ok),
            ('is_percentage_thresh_ok', is_percentage_thresh_ok),
        ]))

        if not are_thresholds_ok:
            logger.info('wal-e backup size diff is over threshold, falling back '
                        'to other means of restore: %s', human_context)
        else:
            logger.info('Thresholds are OK, using wal-e basebackup: %s', human_context)
        return are_thresholds_ok
def should_use_s3_to_create_replica ( self ) : threshold_megabytes = self . wal_e . threshold_mb threshold_percent = self . wal_e . threshold_pct try : cmd = self . wal_e . cmd + [ 'backup-list' , '--detail' , 'LATEST' ] logger . debug ( 'calling %r' , cmd ) wale_output = subprocess . check_output ( cmd ) reader = csv . DictReader ( wale_output . decode ( 'utf-8' ) . splitlines ( ) , dialect = 'excel-tab' ) rows = list ( reader ) if not len ( rows ) : logger . warning ( 'wal-e did not find any backups' ) return False if len ( rows ) > 1 : logger . warning ( 'wal-e returned more than one row of backups: %r' , rows ) return False backup_info = rows [ 0 ] except subprocess . CalledProcessError : logger . exception ( "could not query wal-e latest backup" ) return None try : backup_size = int ( backup_info [ 'expanded_size_bytes' ] ) backup_start_segment = backup_info [ 'wal_segment_backup_start' ] backup_start_offset = backup_info [ 'wal_segment_offset_backup_start' ] except KeyError : logger . exception ( "unable to get some of WALE backup parameters" ) return None lsn_segment = backup_start_segment [ 8 : 16 ] lsn_offset = hex ( ( int ( backup_start_segment [ 16 : 32 ] , 16 ) << 24 ) + int ( backup_start_offset ) ) [ 2 : - 1 ] backup_start_lsn = '{0}/{1}' . format ( lsn_segment , lsn_offset ) diff_in_bytes = backup_size attempts_no = 0 while True : if self . master_connection : try : with psycopg2 . connect ( self . master_connection ) as con : if con . server_version >= 100000 : wal_name = 'wal' lsn_name = 'lsn' else : wal_name = 'xlog' lsn_name = 'location' con . autocommit = True with con . cursor ( ) as cur : cur . execute ( ( "SELECT CASE WHEN pg_catalog.pg_is_in_recovery()" " THEN GREATEST(pg_catalog.pg_{0}_{1}_diff(COALESCE(" "pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint, " "pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), %s)::bigint)" " ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), %s)::bigint" " END" ) . format ( wal_name , lsn_name ) , ( backup_start_lsn , backup_start_lsn , backup_start_lsn ) ) diff_in_bytes = int ( cur . fetchone ( ) [ 0 ] ) except psycopg2 . Error : logger . exception ( 'could not determine difference with the master location' ) if attempts_no < self . retries : attempts_no = attempts_no + 1 time . sleep ( RETRY_SLEEP_INTERVAL ) continue else : if not self . no_master : return False logger . info ( "continue with base backup from S3 since master is not available" ) diff_in_bytes = 0 break else : diff_in_bytes = 0 break is_size_thresh_ok = diff_in_bytes < int ( threshold_megabytes ) * 1048576 threshold_pct_bytes = backup_size * threshold_percent / 100.0 is_percentage_thresh_ok = float ( diff_in_bytes ) < int ( threshold_pct_bytes ) are_thresholds_ok = is_size_thresh_ok and is_percentage_thresh_ok class Size ( object ) : def __init__ ( self , n_bytes , prefix = None ) : self . n_bytes = n_bytes self . prefix = prefix def __repr__ ( self ) : if self . prefix is not None : n_bytes = size_as_bytes ( self . n_bytes , self . prefix ) else : n_bytes = self . n_bytes return repr_size ( n_bytes ) class HumanContext ( object ) : def __init__ ( self , items ) : self . items = items def __repr__ ( self ) : return ', ' . join ( '{}={!r}' . format ( key , value ) for key , value in self . items ) human_context = repr ( HumanContext ( [ ( 'threshold_size' , Size ( threshold_megabytes , 'M' ) ) , ( 'threshold_percent' , threshold_percent ) , ( 'threshold_percent_size' , Size ( threshold_pct_bytes ) ) , ( 'backup_size' , Size ( backup_size ) ) , ( 'backup_diff' , Size ( diff_in_bytes ) ) , ( 'is_size_thresh_ok' , is_size_thresh_ok ) , ( 'is_percentage_thresh_ok' , is_percentage_thresh_ok ) , ] ) ) if not are_thresholds_ok : logger . info ( 'wal-e backup size diff is over threshold, falling back ' 'to other means of restore: %s' , human_context ) else : logger . info ( 'Thresholds are OK, using wal-e basebackup: %s' , human_context ) return are_thresholds_ok
determine whether it makes sense to use S3 and not pg_basebackup