<s> """<STR_LIT>""" <EOL> from __future__ import with_statement <EOL> import os . path <EOL> import hashlib <EOL> from os . path import join as pjoin <EOL> from libcloud . utils . py3 import httplib <EOL> from libcloud . utils . py3 import next <EOL> from libcloud . utils . py3 import b <EOL> import libcloud . utils . files <EOL> from libcloud . common . types import LibcloudError <EOL> from libcloud . common . base import ConnectionUserAndKey , BaseDriver <EOL> from libcloud . storage . types import ObjectDoesNotExistError <EOL> __all__ = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> CHUNK_SIZE = <NUM_LIT> <EOL> DEFAULT_CONTENT_TYPE = '<STR_LIT>' <EOL> class Object ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , size , hash , extra , meta_data , container , <EOL> driver ) : <EOL> """<STR_LIT>""" <EOL> self . name = name <EOL> self . size = size <EOL> self . hash = hash <EOL> self . container = container <EOL> self . extra = extra or { } <EOL> self . meta_data = meta_data or { } <EOL> self . driver = driver <EOL> def get_cdn_url ( self ) : <EOL> return self . driver . get_object_cdn_url ( obj = self ) <EOL> def enable_cdn ( self , ** kwargs ) : <EOL> return self . driver . enable_object_cdn ( obj = self , ** kwargs ) <EOL> def download ( self , destination_path , overwrite_existing = False , <EOL> delete_on_failure = True ) : <EOL> return self . driver . download_object ( self , destination_path , <EOL> overwrite_existing , <EOL> delete_on_failure ) <EOL> def as_stream ( self , chunk_size = None ) : <EOL> return self . driver . download_object_as_stream ( self , chunk_size ) <EOL> def delete ( self ) : <EOL> return self . driver . delete_object ( self ) <EOL> def __repr__ ( self ) : <EOL> return ( '<STR_LIT>' % <EOL> ( self . name , self . size , self . hash , self . driver . name ) ) <EOL> class Container ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , extra , driver ) : <EOL> """<STR_LIT>""" <EOL> self . name = name <EOL> self . extra = extra or { } <EOL> self . driver = driver <EOL> def iterate_objects ( self ) : <EOL> return self . driver . iterate_container_objects ( container = self ) <EOL> def list_objects ( self ) : <EOL> return self . driver . list_container_objects ( container = self ) <EOL> def get_cdn_url ( self ) : <EOL> return self . driver . get_container_cdn_url ( container = self ) <EOL> def enable_cdn ( self , ** kwargs ) : <EOL> return self . driver . enable_container_cdn ( container = self , ** kwargs ) <EOL> def get_object ( self , object_name ) : <EOL> return self . driver . get_object ( container_name = self . name , <EOL> object_name = object_name ) <EOL> def upload_object ( self , file_path , object_name , extra = None , ** kwargs ) : <EOL> return self . driver . upload_object ( <EOL> file_path , self , object_name , extra = extra , ** kwargs ) <EOL> def upload_object_via_stream ( self , iterator , object_name , extra = None , <EOL> ** kwargs ) : <EOL> return self . driver . upload_object_via_stream ( <EOL> iterator , self , object_name , extra = extra , ** kwargs ) <EOL> def download_object ( self , obj , destination_path , overwrite_existing = False , <EOL> delete_on_failure = True ) : <EOL> return self . driver . download_object ( <EOL> obj , destination_path , overwrite_existing = overwrite_existing , <EOL> delete_on_failure = delete_on_failure ) <EOL> def download_object_as_stream ( self , obj , chunk_size = None ) : <EOL> return self . driver . download_object_as_stream ( obj , chunk_size ) <EOL> def delete_object ( self , obj ) : <EOL> return self . driver . delete_object ( obj ) <EOL> def delete ( self ) : <EOL> return self . driver . delete_container ( self ) <EOL> def __repr__ ( self ) : <EOL> return ( '<STR_LIT>' <EOL> % ( self . name , self . driver . name ) ) <EOL> class StorageDriver ( BaseDriver ) : <EOL> """<STR_LIT>""" <EOL> connectionCls = ConnectionUserAndKey <EOL> name = None <EOL> hash_type = '<STR_LIT>' <EOL> supports_chunked_encoding = False <EOL> strict_mode = False <EOL> def iterate_containers ( self ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def list_containers ( self ) : <EOL> """<STR_LIT>""" <EOL> return list ( self . iterate_containers ( ) ) <EOL> def iterate_container_objects ( self , container ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def list_container_objects ( self , container ) : <EOL> """<STR_LIT>""" <EOL> return list ( self . iterate_container_objects ( container ) ) <EOL> def get_container ( self , container_name ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def get_container_cdn_url ( self , container ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def get_object ( self , container_name , object_name ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def get_object_cdn_url ( self , obj ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def enable_container_cdn ( self , container ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def enable_object_cdn ( self , obj ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def download_object ( self , obj , destination_path , overwrite_existing = False , <EOL> delete_on_failure = True ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def download_object_as_stream ( self , obj , chunk_size = None ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def upload_object ( self , file_path , container , object_name , extra = None , <EOL> verify_hash = True , headers = None ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def upload_object_via_stream ( self , iterator , container , <EOL> object_name , <EOL> extra = None , <EOL> headers = None ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def delete_object ( self , obj ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def create_container ( self , container_name ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def delete_container ( self , container ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( <EOL> '<STR_LIT>' ) <EOL> def _get_object ( self , obj , callback , callback_kwargs , response , <EOL> success_status_code = None ) : <EOL> """<STR_LIT>""" <EOL> success_status_code = success_status_code or httplib . OK <EOL> if response . status == success_status_code : <EOL> return callback ( ** callback_kwargs ) <EOL> elif response . status == httplib . NOT_FOUND : <EOL> raise ObjectDoesNotExistError ( object_name = obj . name , <EOL> value = '<STR_LIT>' , driver = self ) <EOL> raise LibcloudError ( value = '<STR_LIT>' % <EOL> ( response . status ) , <EOL> driver = self ) <EOL> def _save_object ( self , response , obj , destination_path , <EOL> overwrite_existing = False , delete_on_failure = True , <EOL> chunk_size = None ) : <EOL> """<STR_LIT>""" <EOL> chunk_size = chunk_size or CHUNK_SIZE <EOL> base_name = os . path . basename ( destination_path ) <EOL> if not base_name and not os . path . exists ( destination_path ) : <EOL> raise LibcloudError ( <EOL> value = '<STR_LIT>' % ( destination_path ) , <EOL> driver = self ) <EOL> if not base_name : <EOL> file_path = pjoin ( destination_path , obj . name ) <EOL> else : <EOL> file_path = destination_path <EOL> if os . path . exists ( file_path ) and not overwrite_existing : <EOL> raise LibcloudError ( <EOL> value = '<STR_LIT>' % ( file_path ) + <EOL> '<STR_LIT>' , <EOL> driver = self ) <EOL> stream = libcloud . utils . files . read_in_chunks ( response , chunk_size ) <EOL> try : <EOL> data_read = next ( stream ) <EOL> except StopIteration : <EOL> return False <EOL> bytes_transferred = <NUM_LIT:0> <EOL> with open ( file_path , '<STR_LIT:wb>' ) as file_handle : <EOL> while len ( data_read ) > <NUM_LIT:0> : <EOL> file_handle . write ( b ( data_read ) ) <EOL> bytes_transferred += len ( data_read ) <EOL> try : <EOL> data_read = next ( stream ) <EOL> except StopIteration : <EOL> data_read = '<STR_LIT>' <EOL> if int ( obj . size ) != int ( bytes_transferred ) : <EOL> if delete_on_failure : <EOL> try : <EOL> os . unlink ( file_path ) <EOL> except Exception : <EOL> pass <EOL> return False <EOL> return True <EOL> def _upload_object ( self , object_name , content_type , upload_func , <EOL> upload_func_kwargs , request_path , request_method = '<STR_LIT>' , <EOL> headers = None , file_path = None , iterator = None ) : <EOL> """<STR_LIT>""" <EOL> headers = headers or { } <EOL> if file_path and not os . path . exists ( file_path ) : <EOL> raise OSError ( '<STR_LIT>' % ( file_path ) ) <EOL> if iterator is not None and not hasattr ( iterator , '<STR_LIT>' ) and not hasattr ( iterator , '<STR_LIT>' ) : <EOL> raise AttributeError ( '<STR_LIT>' + <EOL> '<STR_LIT>' ) <EOL> if not content_type : <EOL> if file_path : <EOL> name = file_path <EOL> else : <EOL> name = object_name <EOL> content_type , _ = libcloud . utils . files . guess_file_mime_type ( name ) <EOL> if not content_type : <EOL> if self . strict_mode : <EOL> raise AttributeError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> content_type = DEFAULT_CONTENT_TYPE <EOL> file_size = None <EOL> if iterator : <EOL> if self . supports_chunked_encoding : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> upload_func_kwargs [ '<STR_LIT>' ] = True <EOL> else : <EOL> iterator = libcloud . utils . files . read_in_chunks ( <EOL> iterator = iterator ) <EOL> data = libcloud . utils . files . exhaust_iterator ( iterator = iterator ) <EOL> file_size = len ( data ) <EOL> upload_func_kwargs [ '<STR_LIT:data>' ] = data <EOL> else : <EOL> file_size = os . path . getsize ( file_path ) <EOL> upload_func_kwargs [ '<STR_LIT>' ] = False <EOL> if file_size is not None and '<STR_LIT>' not in headers : <EOL> headers [ '<STR_LIT>' ] = file_size <EOL> headers [ '<STR_LIT:Content-Type>' ] = content_type <EOL> response = self . connection . request ( request_path , <EOL> method = request_method , data = None , <EOL> headers = headers , raw = True ) <EOL> upload_func_kwargs [ '<STR_LIT>' ] = response <EOL> success , data_hash , bytes_transferred = upload_func ( <EOL> ** upload_func_kwargs ) <EOL> if not success : <EOL> raise LibcloudError ( <EOL> value = '<STR_LIT>' , driver = self ) <EOL> result_dict = { '<STR_LIT>' : response , '<STR_LIT>' : data_hash , <EOL> '<STR_LIT>' : bytes_transferred } <EOL> return result_dict <EOL> def _upload_data ( self , response , data , calculate_hash = True ) : <EOL> """<STR_LIT>""" <EOL> bytes_transferred = <NUM_LIT:0> <EOL> data_hash = None <EOL> if calculate_hash : <EOL> data_hash = self . _get_hash_function ( ) <EOL> data_hash . update ( b ( data ) ) <EOL> try : <EOL> response . connection . connection . send ( b ( data ) ) <EOL> except Exception : <EOL> return False , None , bytes_transferred <EOL> bytes_transferred = len ( data ) <EOL> if calculate_hash : <EOL> data_hash = data_hash . hexdigest ( ) <EOL> return True , data_hash , bytes_transferred <EOL> def _stream_data ( self , response , iterator , chunked = False , <EOL> calculate_hash = True , chunk_size = None , data = None ) : <EOL> """<STR_LIT>""" <EOL> chunk_size = chunk_size or CHUNK_SIZE <EOL> data_hash = None <EOL> if calculate_hash : <EOL> data_hash = self . _get_hash_function ( ) <EOL> generator = libcloud . utils . files . read_in_chunks ( iterator , chunk_size , <EOL> fill_size = True ) <EOL> bytes_transferred = <NUM_LIT:0> <EOL> try : <EOL> chunk = next ( generator ) <EOL> except StopIteration : <EOL> chunk = '<STR_LIT>' <EOL> if chunked : <EOL> response . connection . connection . send ( b ( '<STR_LIT>' % <EOL> ( len ( chunk ) ) ) ) <EOL> response . connection . connection . send ( chunk ) <EOL> response . connection . connection . send ( b ( '<STR_LIT:\r\n>' ) ) <EOL> response . connection . connection . send ( b ( '<STR_LIT>' ) ) <EOL> else : <EOL> response . connection . connection . send ( chunk ) <EOL> return True , data_hash . hexdigest ( ) , bytes_transferred <EOL> while len ( chunk ) > <NUM_LIT:0> : <EOL> try : <EOL> if chunked : <EOL> response . connection . connection . send ( b ( '<STR_LIT>' % <EOL> ( len ( chunk ) ) ) ) <EOL> response . connection . connection . send ( b ( chunk ) ) <EOL> response . connection . connection . send ( b ( '<STR_LIT:\r\n>' ) ) <EOL> else : <EOL> response . connection . connection . send ( b ( chunk ) ) <EOL> except Exception : <EOL> return False , None , bytes_transferred <EOL> bytes_transferred += len ( chunk ) <EOL> if calculate_hash : <EOL> data_hash . update ( b ( chunk ) ) <EOL> try : <EOL> chunk = next ( generator ) <EOL> except StopIteration : <EOL> chunk = '<STR_LIT>' <EOL> if chunked : <EOL> response . connection . connection . send ( b ( '<STR_LIT>' ) ) <EOL> if calculate_hash : <EOL> data_hash = data_hash . hexdigest ( ) <EOL> return True , data_hash , bytes_transferred <EOL> def _upload_file ( self , response , file_path , chunked = False , <EOL> calculate_hash = True ) : <EOL> """<STR_LIT>""" <EOL> with open ( file_path , '<STR_LIT:rb>' ) as file_handle : <EOL> success , data_hash , bytes_transferred = ( <EOL> self . _stream_data ( <EOL> response = response , <EOL> iterator = iter ( file_handle ) , <EOL> chunked = chunked , <EOL> calculate_hash = calculate_hash ) ) <EOL> return success , data_hash , bytes_transferred <EOL> def _get_hash_function ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> func = getattr ( hashlib , self . hash_type ) ( ) <EOL> except AttributeError : <EOL> raise RuntimeError ( '<STR_LIT>' % <EOL> ( self . hash_type ) ) <EOL> return func </s>
