def do_lzop_get(creds, uri, path, decrypt, do_retry=True):
 assert uri.endswith('.lzo'), 'Expect an lzop-compressed file'
 def log_wal_fetch_failures_on_error(exc_tup, exc_processor_cxt):
  def standard_detail_message(prefix=''):
   return (prefix + '  There have been {n} attempts to fetch wal file {uri} so far.'.format(n=exc_processor_cxt, uri=uri))
  (typ, value, tb) = exc_tup
  del exc_tup
  if issubclass(typ, socket.error):
   socketmsg = (value[1] if isinstance(value, tuple) else value)
   logger.info(msg='Retrying fetch because of a socket error', detail=standard_detail_message("The socket error's message is '{0}'.".format(socketmsg)))
  else:
   logger.warning(msg='retrying WAL file fetch from unexpected exception', detail=standard_detail_message('The exception type is {etype} and its value is {evalue} and its traceback is {etraceback}'.format(etype=typ, evalue=value, etraceback=''.join(traceback.format_tb(tb)))))
  del tb
 def download():
  with files.DeleteOnError(path) as decomp_out:
   with get_download_pipeline(PIPE, decomp_out.f, decrypt) as pl:
    conn = calling_format.connect(creds)
    g = gevent.spawn(write_and_return_error, uri, conn, pl.stdin)
    try:
     exc = g.get()
     if (exc is not None):
      raise exc
    except ClientException as e:
     if (e.http_status == 404):
      pl.abort()
      logger.warning(msg='could no longer locate object while performing wal restore', detail='The absolute URI that could not be located is {uri}.'.format(uri=uri), hint='This can be normal when Postgres is trying to detect what timelines are available during restoration.')
      decomp_out.remove_regardless = True
      return False
     else:
      raise
   logger.info(msg='completed download and decompression', detail='Downloaded and decompressed "{uri}" to "{path}"'.format(uri=uri, path=path))
  return True
 if do_retry:
  download = retry(retry_with_count(log_wal_fetch_failures_on_error))(download)
 return download()

def do_lzop_get ( creds , uri , path , decrypt , do_retry = True ) : assert uri . endswith ( '.lzo' ) , 'Expect an lzop-compressed file' def log_wal_fetch_failures_on_error ( exc_tup , exc_processor_cxt ) : def standard_detail_message ( prefix = '' ) : return ( prefix + '  There have been {n} attempts to fetch wal file {uri} so far.' . format ( n = exc_processor_cxt , uri = uri ) ) ( typ , value , tb ) = exc_tup del exc_tup if issubclass ( typ , socket . error ) : socketmsg = ( value [ 1 ] if isinstance ( value , tuple ) else value ) logger . info ( msg = 'Retrying fetch because of a socket error' , detail = standard_detail_message ( "The socket error's message is '{0}'." . format ( socketmsg ) ) ) else : logger . warning ( msg = 'retrying WAL file fetch from unexpected exception' , detail = standard_detail_message ( 'The exception type is {etype} and its value is {evalue} and its traceback is {etraceback}' . format ( etype = typ , evalue = value , etraceback = '' . join ( traceback . format_tb ( tb ) ) ) ) ) del tb def download ( ) : with files . DeleteOnError ( path ) as decomp_out : with get_download_pipeline ( PIPE , decomp_out . f , decrypt ) as pl : conn = calling_format . connect ( creds ) g = gevent . spawn ( write_and_return_error , uri , conn , pl . stdin ) try : exc = g . get ( ) if ( exc is not None ) : raise exc except ClientException as e : if ( e . http_status == 404 ) : pl . abort ( ) logger . warning ( msg = 'could no longer locate object while performing wal restore' , detail = 'The absolute URI that could not be located is {uri}.' . format ( uri = uri ) , hint = 'This can be normal when Postgres is trying to detect what timelines are available during restoration.' ) decomp_out . remove_regardless = True return False else : raise logger . info ( msg = 'completed download and decompression' , detail = 'Downloaded and decompressed "{uri}" to "{path}"' . format ( uri = uri , path = path ) ) return True if do_retry : download = retry ( retry_with_count ( log_wal_fetch_failures_on_error ) ) ( download ) return download ( )
get and decompress an s3 or wabs url this streams the content directly to lzop; the compressed version is never stored on disk .