<s> """<STR_LIT>""" <EOL> import logging <EOL> import hashlib <EOL> import httplib <EOL> import tempfile <EOL> import urlparse <EOL> from glance . common import cfg <EOL> from glance . common import exception <EOL> import glance . store <EOL> import glance . store . base <EOL> import glance . store . location <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> class StoreLocation ( glance . store . location . StoreLocation ) : <EOL> """<STR_LIT>""" <EOL> def process_specs ( self ) : <EOL> self . scheme = self . specs . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . accesskey = self . specs . get ( '<STR_LIT>' ) <EOL> self . secretkey = self . specs . get ( '<STR_LIT>' ) <EOL> s3_host = self . specs . get ( '<STR_LIT>' ) <EOL> self . bucket = self . specs . get ( '<STR_LIT>' ) <EOL> self . key = self . specs . get ( '<STR_LIT:key>' ) <EOL> if s3_host . startswith ( '<STR_LIT>' ) : <EOL> self . scheme = '<STR_LIT>' <EOL> s3_host = s3_host [ <NUM_LIT:8> : ] . strip ( '<STR_LIT:/>' ) <EOL> elif s3_host . startswith ( '<STR_LIT>' ) : <EOL> s3_host = s3_host [ <NUM_LIT:7> : ] . strip ( '<STR_LIT:/>' ) <EOL> self . s3serviceurl = s3_host . strip ( '<STR_LIT:/>' ) <EOL> def _get_credstring ( self ) : <EOL> if self . accesskey : <EOL> return '<STR_LIT>' % ( self . accesskey , self . secretkey ) <EOL> return '<STR_LIT>' <EOL> def get_uri ( self ) : <EOL> return "<STR_LIT>" % ( <EOL> self . scheme , <EOL> self . _get_credstring ( ) , <EOL> self . s3serviceurl , <EOL> self . bucket , <EOL> self . key ) <EOL> def parse_uri ( self , uri ) : <EOL> """<STR_LIT>""" <EOL> if uri . count ( '<STR_LIT>' ) != <NUM_LIT:1> : <EOL> reason = _ ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> ) <EOL> raise exception . BadStoreUri ( uri , reason ) <EOL> pieces = urlparse . urlparse ( uri ) <EOL> assert pieces . scheme in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . scheme = pieces . scheme <EOL> path = pieces . path . strip ( '<STR_LIT:/>' ) <EOL> netloc = pieces . netloc . strip ( '<STR_LIT:/>' ) <EOL> entire_path = ( netloc + '<STR_LIT:/>' + path ) . strip ( '<STR_LIT:/>' ) <EOL> if '<STR_LIT:@>' in uri : <EOL> creds , path = entire_path . split ( '<STR_LIT:@>' ) <EOL> cred_parts = creds . split ( '<STR_LIT::>' ) <EOL> try : <EOL> access_key = cred_parts [ <NUM_LIT:0> ] <EOL> secret_key = cred_parts [ <NUM_LIT:1> ] <EOL> access_key = access_key . encode ( '<STR_LIT:utf-8>' ) <EOL> secret_key = secret_key . encode ( '<STR_LIT:utf-8>' ) <EOL> self . accesskey = access_key <EOL> self . secretkey = secret_key <EOL> except IndexError : <EOL> reason = _ ( "<STR_LIT>" ) % creds <EOL> raise exception . BadStoreUri ( uri , reason ) <EOL> else : <EOL> self . accesskey = None <EOL> path = entire_path <EOL> try : <EOL> path_parts = path . split ( '<STR_LIT:/>' ) <EOL> self . key = path_parts . pop ( ) <EOL> self . bucket = path_parts . pop ( ) <EOL> if len ( path_parts ) > <NUM_LIT:0> : <EOL> self . s3serviceurl = '<STR_LIT:/>' . join ( path_parts ) . strip ( '<STR_LIT:/>' ) <EOL> else : <EOL> reason = _ ( "<STR_LIT>" ) <EOL> raise exception . BadStoreUri ( uri , reason ) <EOL> except IndexError : <EOL> reason = _ ( "<STR_LIT>" ) <EOL> raise exception . BadStoreUri ( uri , reason ) <EOL> class ChunkedFile ( object ) : <EOL> """<STR_LIT>""" <EOL> CHUNKSIZE = <NUM_LIT> <EOL> def __init__ ( self , fp ) : <EOL> self . fp = fp <EOL> def __iter__ ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> while True : <EOL> chunk = self . fp . read ( ChunkedFile . CHUNKSIZE ) <EOL> if chunk : <EOL> yield chunk <EOL> else : <EOL> break <EOL> finally : <EOL> self . close ( ) <EOL> def getvalue ( self ) : <EOL> """<STR_LIT>""" <EOL> data = "<STR_LIT>" <EOL> self . len = <NUM_LIT:0> <EOL> for chunk in self : <EOL> read_bytes = len ( chunk ) <EOL> data = data + chunk <EOL> self . len = self . len + read_bytes <EOL> return data <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . fp : <EOL> self . fp . close ( ) <EOL> self . fp = None <EOL> class Store ( glance . store . base . Store ) : <EOL> """<STR_LIT>""" <EOL> EXAMPLE_URL = "<STR_LIT>" <EOL> opts = [ <EOL> cfg . StrOpt ( '<STR_LIT>' ) , <EOL> cfg . StrOpt ( '<STR_LIT>' ) , <EOL> cfg . StrOpt ( '<STR_LIT>' ) , <EOL> cfg . StrOpt ( '<STR_LIT>' ) , <EOL> cfg . StrOpt ( '<STR_LIT>' ) , <EOL> cfg . BoolOpt ( '<STR_LIT>' , default = False ) , <EOL> ] <EOL> def configure_add ( self ) : <EOL> """<STR_LIT>""" <EOL> self . conf . register_opts ( self . opts ) <EOL> self . s3_host = self . _option_get ( '<STR_LIT>' ) <EOL> access_key = self . _option_get ( '<STR_LIT>' ) <EOL> secret_key = self . _option_get ( '<STR_LIT>' ) <EOL> self . access_key = access_key . encode ( '<STR_LIT:utf-8>' ) <EOL> self . secret_key = secret_key . encode ( '<STR_LIT:utf-8>' ) <EOL> self . bucket = self . _option_get ( '<STR_LIT>' ) <EOL> self . scheme = '<STR_LIT>' <EOL> if self . s3_host . startswith ( '<STR_LIT>' ) : <EOL> self . scheme = '<STR_LIT>' <EOL> self . full_s3_host = self . s3_host <EOL> elif self . s3_host . startswith ( '<STR_LIT>' ) : <EOL> self . full_s3_host = self . s3_host <EOL> else : <EOL> self . full_s3_host = '<STR_LIT>' + self . s3_host <EOL> self . s3_store_object_buffer_dir = self . conf . s3_store_object_buffer_dir <EOL> def _option_get ( self , param ) : <EOL> result = getattr ( self . conf , param ) <EOL> if not result : <EOL> reason = _ ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % locals ( ) <EOL> logger . error ( reason ) <EOL> raise exception . BadStoreConfiguration ( store_name = "<STR_LIT>" , <EOL> reason = reason ) <EOL> return result <EOL> def get ( self , location ) : <EOL> """<STR_LIT>""" <EOL> key = self . _retrieve_key ( location ) <EOL> key . BufferSize = self . CHUNKSIZE <EOL> return ( ChunkedFile ( key ) , key . size ) <EOL> def get_size ( self , location ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> key = self . _retrieve_key ( location ) <EOL> return key . size <EOL> except Exception : <EOL> return <NUM_LIT:0> <EOL> def _retrieve_key ( self , location ) : <EOL> loc = location . store_location <EOL> from boto . s3 . connection import S3Connection <EOL> s3_conn = S3Connection ( loc . accesskey , loc . secretkey , <EOL> host = loc . s3serviceurl , <EOL> is_secure = ( loc . scheme == '<STR_LIT>' ) ) <EOL> bucket_obj = get_bucket ( s3_conn , loc . bucket ) <EOL> key = get_key ( bucket_obj , loc . key ) <EOL> msg = _ ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) % ( { '<STR_LIT>' : loc . s3serviceurl , <EOL> '<STR_LIT>' : loc . accesskey , '<STR_LIT>' : loc . bucket , <EOL> '<STR_LIT>' : loc . key } ) <EOL> logger . debug ( msg ) <EOL> return key <EOL> def add ( self , image_id , image_file , image_size ) : <EOL> """<STR_LIT>""" <EOL> from boto . s3 . connection import S3Connection <EOL> loc = StoreLocation ( { '<STR_LIT>' : self . scheme , <EOL> '<STR_LIT>' : self . bucket , <EOL> '<STR_LIT:key>' : image_id , <EOL> '<STR_LIT>' : self . full_s3_host , <EOL> '<STR_LIT>' : self . access_key , <EOL> '<STR_LIT>' : self . secret_key } ) <EOL> s3_conn = S3Connection ( loc . accesskey , loc . secretkey , <EOL> host = loc . s3serviceurl , <EOL> is_secure = ( loc . scheme == '<STR_LIT>' ) ) <EOL> create_bucket_if_missing ( self . bucket , s3_conn , self . conf ) <EOL> bucket_obj = get_bucket ( s3_conn , self . bucket ) <EOL> obj_name = str ( image_id ) <EOL> key = bucket_obj . get_key ( obj_name ) <EOL> if key and key . exists ( ) : <EOL> raise exception . Duplicate ( _ ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % loc . get_uri ( ) ) <EOL> msg = _ ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) % ( { '<STR_LIT>' : self . s3_host , <EOL> '<STR_LIT>' : self . access_key , '<STR_LIT>' : self . bucket , <EOL> '<STR_LIT>' : obj_name } ) <EOL> logger . debug ( msg ) <EOL> key = bucket_obj . new_key ( obj_name ) <EOL> msg = _ ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % loc . get_uri ( ) <EOL> logger . debug ( msg ) <EOL> tmpdir = self . s3_store_object_buffer_dir <EOL> temp_file = tempfile . NamedTemporaryFile ( dir = tmpdir ) <EOL> checksum = hashlib . md5 ( ) <EOL> chunk = image_file . read ( self . CHUNKSIZE ) <EOL> while chunk : <EOL> checksum . update ( chunk ) <EOL> temp_file . write ( chunk ) <EOL> chunk = image_file . read ( self . CHUNKSIZE ) <EOL> temp_file . flush ( ) <EOL> msg = _ ( "<STR_LIT>" ) % loc . get_uri ( ) <EOL> logger . debug ( msg ) <EOL> key . set_contents_from_file ( open ( temp_file . name , '<STR_LIT>' ) , replace = False ) <EOL> size = key . size <EOL> checksum_hex = checksum . hexdigest ( ) <EOL> logger . debug ( _ ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % locals ( ) ) <EOL> return ( loc . get_uri ( ) , size , checksum_hex ) <EOL> def delete ( self , location ) : <EOL> """<STR_LIT>""" <EOL> loc = location . store_location <EOL> from boto . s3 . connection import S3Connection <EOL> s3_conn = S3Connection ( loc . accesskey , loc . secretkey , <EOL> host = loc . s3serviceurl , <EOL> is_secure = ( loc . scheme == '<STR_LIT>' ) ) <EOL> bucket_obj = get_bucket ( s3_conn , loc . bucket ) <EOL> key = get_key ( bucket_obj , loc . key ) <EOL> msg = _ ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) % ( { '<STR_LIT>' : loc . s3serviceurl , <EOL> '<STR_LIT>' : loc . accesskey , '<STR_LIT>' : loc . bucket , <EOL> '<STR_LIT>' : loc . key } ) <EOL> logger . debug ( msg ) <EOL> return key . delete ( ) <EOL> def get_bucket ( conn , bucket_id ) : <EOL> """<STR_LIT>""" <EOL> bucket = conn . get_bucket ( bucket_id ) <EOL> if not bucket : <EOL> msg = _ ( "<STR_LIT>" ) % locals ( ) <EOL> logger . error ( msg ) <EOL> raise exception . NotFound ( msg ) <EOL> return bucket <EOL> def create_bucket_if_missing ( bucket , s3_conn , conf ) : <EOL> """<STR_LIT>""" <EOL> from boto . exception import S3ResponseError <EOL> try : <EOL> s3_conn . get_bucket ( bucket ) <EOL> except S3ResponseError , e : <EOL> if e . status == httplib . NOT_FOUND : <EOL> if conf . s3_store_create_bucket_on_put : <EOL> try : <EOL> s3_conn . create_bucket ( bucket ) <EOL> except S3ResponseError , e : <EOL> msg = ( "<STR_LIT>" <EOL> "<STR_LIT>" % locals ( ) ) <EOL> raise glance . store . BackendException ( msg ) <EOL> else : <EOL> msg = ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % locals ( ) ) <EOL> raise glance . store . BackendException ( msg ) <EOL> def get_key ( bucket , obj ) : <EOL> """<STR_LIT>""" <EOL> key = bucket . get_key ( obj ) <EOL> if not key or not key . exists ( ) : <EOL> msg = _ ( "<STR_LIT>" ) % locals ( ) <EOL> logger . error ( msg ) <EOL> raise exception . NotFound ( msg ) <EOL> return key <EOL> glance . store . register_store ( __name__ , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) </s>
