<s> """<STR_LIT>""" <EOL> import hashlib <EOL> import os <EOL> import re <EOL> import time <EOL> from oslo_concurrency import lockutils <EOL> from oslo_concurrency import processutils <EOL> from oslo_config import cfg <EOL> from oslo_log import log as logging <EOL> from oslo_serialization import jsonutils <EOL> from oslo_utils import fileutils <EOL> import nova . conf <EOL> from nova . i18n import _LE <EOL> from nova . i18n import _LI <EOL> from nova . i18n import _LW <EOL> from nova import utils <EOL> from nova . virt import imagecache <EOL> from nova . virt . libvirt import utils as libvirt_utils <EOL> LOG = logging . getLogger ( __name__ ) <EOL> imagecache_opts = [ <EOL> cfg . StrOpt ( '<STR_LIT>' , <EOL> default = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> cfg . BoolOpt ( '<STR_LIT>' , <EOL> default = True , <EOL> deprecated_for_removal = True , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> cfg . IntOpt ( '<STR_LIT>' , <EOL> default = <NUM_LIT> , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> cfg . BoolOpt ( '<STR_LIT>' , <EOL> default = False , <EOL> help = '<STR_LIT>' ) , <EOL> cfg . IntOpt ( '<STR_LIT>' , <EOL> default = <NUM_LIT> , <EOL> help = '<STR_LIT>' ) , <EOL> ] <EOL> CONF = nova . conf . CONF <EOL> CONF . register_opts ( imagecache_opts , '<STR_LIT>' ) <EOL> def get_cache_fname ( images , key ) : <EOL> """<STR_LIT>""" <EOL> image_id = str ( images [ key ] ) <EOL> if ( ( not CONF . libvirt . remove_unused_kernels and <EOL> key in [ '<STR_LIT>' , '<STR_LIT>' ] ) ) : <EOL> return image_id <EOL> else : <EOL> return hashlib . sha1 ( image_id ) . hexdigest ( ) <EOL> def get_info_filename ( base_path ) : <EOL> """<STR_LIT>""" <EOL> base_file = os . path . basename ( base_path ) <EOL> return ( CONF . libvirt . image_info_filename_pattern <EOL> % { '<STR_LIT:image>' : base_file } ) <EOL> def is_valid_info_file ( path ) : <EOL> """<STR_LIT>""" <EOL> digest_size = hashlib . sha1 ( ) . digestsize * <NUM_LIT:2> <EOL> regexp = ( CONF . libvirt . image_info_filename_pattern <EOL> % { '<STR_LIT:image>' : ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % { '<STR_LIT>' : digest_size } ) } ) <EOL> m = re . match ( regexp , path ) <EOL> if m : <EOL> return True <EOL> return False <EOL> def _read_possible_json ( serialized , info_file ) : <EOL> try : <EOL> d = jsonutils . loads ( serialized ) <EOL> except ValueError as e : <EOL> LOG . error ( _LE ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT:filename>' : info_file , <EOL> '<STR_LIT:error>' : e } ) <EOL> d = { } <EOL> return d <EOL> def read_stored_info ( target , field = None , timestamped = False ) : <EOL> """<STR_LIT>""" <EOL> info_file = get_info_filename ( target ) <EOL> if not os . path . exists ( info_file ) : <EOL> old_filename = target + '<STR_LIT>' <EOL> if field == '<STR_LIT>' and os . path . exists ( old_filename ) : <EOL> with open ( old_filename ) as hash_file : <EOL> hash_value = hash_file . read ( ) <EOL> write_stored_info ( target , field = field , value = hash_value ) <EOL> os . remove ( old_filename ) <EOL> d = { field : hash_value } <EOL> else : <EOL> d = { } <EOL> else : <EOL> lock_name = '<STR_LIT>' % os . path . split ( target ) [ - <NUM_LIT:1> ] <EOL> lock_path = os . path . join ( CONF . instances_path , '<STR_LIT>' ) <EOL> @ utils . synchronized ( lock_name , external = True , lock_path = lock_path ) <EOL> def read_file ( info_file ) : <EOL> LOG . debug ( '<STR_LIT>' , info_file ) <EOL> with open ( info_file , '<STR_LIT:r>' ) as f : <EOL> return f . read ( ) . rstrip ( ) <EOL> serialized = read_file ( info_file ) <EOL> d = _read_possible_json ( serialized , info_file ) <EOL> if field : <EOL> if timestamped : <EOL> return ( d . get ( field , None ) , d . get ( '<STR_LIT>' % field , None ) ) <EOL> else : <EOL> return d . get ( field , None ) <EOL> return d <EOL> def write_stored_info ( target , field = None , value = None ) : <EOL> """<STR_LIT>""" <EOL> if not field : <EOL> return <EOL> info_file = get_info_filename ( target ) <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , info_file ) <EOL> fileutils . ensure_tree ( os . path . dirname ( info_file ) ) <EOL> lock_name = '<STR_LIT>' % os . path . split ( target ) [ - <NUM_LIT:1> ] <EOL> lock_path = os . path . join ( CONF . instances_path , '<STR_LIT>' ) <EOL> @ utils . synchronized ( lock_name , external = True , lock_path = lock_path ) <EOL> def write_file ( info_file , field , value ) : <EOL> d = { } <EOL> if os . path . exists ( info_file ) : <EOL> with open ( info_file , '<STR_LIT:r>' ) as f : <EOL> d = _read_possible_json ( f . read ( ) , info_file ) <EOL> d [ field ] = value <EOL> d [ '<STR_LIT>' % field ] = time . time ( ) <EOL> with open ( info_file , '<STR_LIT:w>' ) as f : <EOL> f . write ( jsonutils . dumps ( d ) ) <EOL> write_file ( info_file , field , value ) <EOL> def _hash_file ( filename ) : <EOL> """<STR_LIT>""" <EOL> checksum = hashlib . sha1 ( ) <EOL> with open ( filename ) as f : <EOL> for chunk in iter ( lambda : f . read ( <NUM_LIT> ) , b'<STR_LIT>' ) : <EOL> checksum . update ( chunk ) <EOL> return checksum . hexdigest ( ) <EOL> def read_stored_checksum ( target , timestamped = True ) : <EOL> """<STR_LIT>""" <EOL> return read_stored_info ( target , field = '<STR_LIT>' , timestamped = timestamped ) <EOL> def write_stored_checksum ( target ) : <EOL> """<STR_LIT>""" <EOL> write_stored_info ( target , field = '<STR_LIT>' , value = _hash_file ( target ) ) <EOL> class ImageCacheManager ( imagecache . ImageCacheManager ) : <EOL> def __init__ ( self ) : <EOL> super ( ImageCacheManager , self ) . __init__ ( ) <EOL> self . lock_path = os . path . join ( CONF . instances_path , '<STR_LIT>' ) <EOL> self . _reset_state ( ) <EOL> def _reset_state ( self ) : <EOL> """<STR_LIT>""" <EOL> self . used_images = { } <EOL> self . image_popularity = { } <EOL> self . instance_names = set ( ) <EOL> self . back_swap_images = set ( ) <EOL> self . used_swap_images = set ( ) <EOL> self . active_base_files = [ ] <EOL> self . corrupt_base_files = [ ] <EOL> self . originals = [ ] <EOL> self . removable_base_files = [ ] <EOL> self . unexplained_images = [ ] <EOL> def _store_image ( self , base_dir , ent , original = False ) : <EOL> """<STR_LIT>""" <EOL> entpath = os . path . join ( base_dir , ent ) <EOL> if os . path . isfile ( entpath ) : <EOL> self . unexplained_images . append ( entpath ) <EOL> if original : <EOL> self . originals . append ( entpath ) <EOL> def _store_swap_image ( self , ent ) : <EOL> """<STR_LIT>""" <EOL> names = ent . split ( '<STR_LIT:_>' ) <EOL> if len ( names ) == <NUM_LIT:2> and names [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> if len ( names [ <NUM_LIT:1> ] ) > <NUM_LIT:0> and names [ <NUM_LIT:1> ] . isdigit ( ) : <EOL> LOG . debug ( '<STR_LIT>' , ent ) <EOL> self . back_swap_images . add ( ent ) <EOL> def _list_base_images ( self , base_dir ) : <EOL> """<STR_LIT>""" <EOL> digest_size = hashlib . sha1 ( ) . digestsize * <NUM_LIT:2> <EOL> for ent in os . listdir ( base_dir ) : <EOL> if len ( ent ) == digest_size : <EOL> self . _store_image ( base_dir , ent , original = True ) <EOL> elif ( len ( ent ) > digest_size + <NUM_LIT:2> and <EOL> ent [ digest_size ] == '<STR_LIT:_>' and <EOL> not is_valid_info_file ( os . path . join ( base_dir , ent ) ) ) : <EOL> self . _store_image ( base_dir , ent , original = False ) <EOL> else : <EOL> self . _store_swap_image ( ent ) <EOL> return { '<STR_LIT>' : self . unexplained_images , <EOL> '<STR_LIT>' : self . originals } <EOL> def _list_backing_images ( self ) : <EOL> """<STR_LIT>""" <EOL> inuse_images = [ ] <EOL> for ent in os . listdir ( CONF . instances_path ) : <EOL> if ent in self . instance_names : <EOL> LOG . debug ( '<STR_LIT>' , ent ) <EOL> disk_path = os . path . join ( CONF . instances_path , ent , '<STR_LIT>' ) <EOL> if os . path . exists ( disk_path ) : <EOL> LOG . debug ( '<STR_LIT>' , ent ) <EOL> try : <EOL> backing_file = libvirt_utils . get_disk_backing_file ( <EOL> disk_path ) <EOL> except processutils . ProcessExecutionError : <EOL> if not os . path . exists ( disk_path ) : <EOL> LOG . debug ( '<STR_LIT>' , <EOL> disk_path ) <EOL> continue <EOL> else : <EOL> raise <EOL> LOG . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> { '<STR_LIT>' : ent , <EOL> '<STR_LIT>' : backing_file } ) <EOL> if backing_file : <EOL> backing_path = os . path . join ( <EOL> CONF . instances_path , <EOL> CONF . image_cache_subdirectory_name , <EOL> backing_file ) <EOL> if backing_path not in inuse_images : <EOL> inuse_images . append ( backing_path ) <EOL> if backing_path in self . unexplained_images : <EOL> LOG . warning ( _LW ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT>' : ent , <EOL> '<STR_LIT>' : backing_file } ) <EOL> self . unexplained_images . remove ( backing_path ) <EOL> return inuse_images <EOL> def _find_base_file ( self , base_dir , fingerprint ) : <EOL> """<STR_LIT>""" <EOL> base_file = os . path . join ( base_dir , fingerprint ) <EOL> if os . path . exists ( base_file ) : <EOL> yield base_file , False , False <EOL> base_file = os . path . join ( base_dir , fingerprint + '<STR_LIT>' ) <EOL> if os . path . exists ( base_file ) : <EOL> yield base_file , True , False <EOL> resize_re = re . compile ( '<STR_LIT>' % fingerprint ) <EOL> for img in self . unexplained_images : <EOL> m = resize_re . match ( img ) <EOL> if m : <EOL> yield img , False , True <EOL> def _verify_checksum ( self , img_id , base_file , create_if_missing = True ) : <EOL> """<STR_LIT>""" <EOL> if not CONF . libvirt . checksum_base_images : <EOL> return None <EOL> lock_name = '<STR_LIT>' % os . path . split ( base_file ) [ - <NUM_LIT:1> ] <EOL> @ utils . synchronized ( lock_name , external = True , lock_path = self . lock_path ) <EOL> def inner_verify_checksum ( ) : <EOL> ( stored_checksum , stored_timestamp ) = read_stored_checksum ( <EOL> base_file , timestamped = True ) <EOL> if stored_checksum : <EOL> if ( stored_timestamp and <EOL> time . time ( ) - stored_timestamp < <EOL> CONF . libvirt . checksum_interval_seconds ) : <EOL> return True <EOL> if not stored_timestamp : <EOL> write_stored_info ( base_file , field = '<STR_LIT>' , <EOL> value = stored_checksum ) <EOL> current_checksum = _hash_file ( base_file ) <EOL> if current_checksum != stored_checksum : <EOL> LOG . error ( _LE ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file } ) <EOL> return False <EOL> else : <EOL> return True <EOL> else : <EOL> LOG . info ( _LI ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file } ) <EOL> if CONF . libvirt . checksum_base_images and create_if_missing : <EOL> LOG . info ( _LI ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file } ) <EOL> write_stored_checksum ( base_file ) <EOL> return None <EOL> return inner_verify_checksum ( ) <EOL> @ staticmethod <EOL> def _get_age_of_file ( base_file ) : <EOL> if not os . path . exists ( base_file ) : <EOL> LOG . debug ( '<STR_LIT>' , base_file ) <EOL> return ( False , <NUM_LIT:0> ) <EOL> mtime = os . path . getmtime ( base_file ) <EOL> age = time . time ( ) - mtime <EOL> return ( True , age ) <EOL> def _remove_old_enough_file ( self , base_file , maxage , remove_sig = True , <EOL> remove_lock = True ) : <EOL> """<STR_LIT>""" <EOL> exists , age = self . _get_age_of_file ( base_file ) <EOL> if not exists : <EOL> return <EOL> lock_file = os . path . split ( base_file ) [ - <NUM_LIT:1> ] <EOL> @ utils . synchronized ( lock_file , external = True , <EOL> lock_path = self . lock_path ) <EOL> def _inner_remove_old_enough_file ( ) : <EOL> exists , age = self . _get_age_of_file ( base_file ) <EOL> if not exists or age < maxage : <EOL> return <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , base_file ) <EOL> try : <EOL> os . remove ( base_file ) <EOL> if remove_sig : <EOL> signature = get_info_filename ( base_file ) <EOL> if os . path . exists ( signature ) : <EOL> os . remove ( signature ) <EOL> except OSError as e : <EOL> LOG . error ( _LE ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT>' : base_file , <EOL> '<STR_LIT:error>' : e } ) <EOL> if age < maxage : <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , <EOL> base_file ) <EOL> else : <EOL> _inner_remove_old_enough_file ( ) <EOL> if remove_lock : <EOL> try : <EOL> lockutils . remove_external_lock_file ( lock_file , <EOL> lock_file_prefix = '<STR_LIT>' , lock_path = self . lock_path ) <EOL> except OSError as e : <EOL> LOG . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> { '<STR_LIT>' : lock_file , <EOL> '<STR_LIT:error>' : e } ) <EOL> def _remove_swap_file ( self , base_file ) : <EOL> """<STR_LIT>""" <EOL> maxage = CONF . remove_unused_original_minimum_age_seconds <EOL> self . _remove_old_enough_file ( base_file , maxage , remove_sig = False , <EOL> remove_lock = False ) <EOL> def _remove_base_file ( self , base_file ) : <EOL> """<STR_LIT>""" <EOL> maxage = CONF . libvirt . remove_unused_resized_minimum_age_seconds <EOL> if base_file in self . originals : <EOL> maxage = CONF . remove_unused_original_minimum_age_seconds <EOL> self . _remove_old_enough_file ( base_file , maxage ) <EOL> def _handle_base_image ( self , img_id , base_file ) : <EOL> """<STR_LIT>""" <EOL> image_bad = False <EOL> image_in_use = False <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file } ) <EOL> if base_file in self . unexplained_images : <EOL> self . unexplained_images . remove ( base_file ) <EOL> if ( base_file and os . path . exists ( base_file ) <EOL> and os . path . isfile ( base_file ) ) : <EOL> checksum_result = self . _verify_checksum ( img_id , base_file ) <EOL> if checksum_result is not None : <EOL> image_bad = not checksum_result <EOL> time . sleep ( <NUM_LIT:0> ) <EOL> if img_id in self . used_images : <EOL> local , remote , instances = self . used_images [ img_id ] <EOL> if local > <NUM_LIT:0> or remote > <NUM_LIT:0> : <EOL> image_in_use = True <EOL> LOG . info ( _LI ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file , <EOL> '<STR_LIT>' : local , <EOL> '<STR_LIT>' : remote } ) <EOL> self . active_base_files . append ( base_file ) <EOL> if not base_file : <EOL> LOG . warning ( _LW ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file , <EOL> '<STR_LIT>' : '<STR_LIT:U+0020>' . join ( instances ) } ) <EOL> if image_bad : <EOL> self . corrupt_base_files . append ( base_file ) <EOL> if base_file : <EOL> if not image_in_use : <EOL> LOG . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file } ) <EOL> self . removable_base_files . append ( base_file ) <EOL> else : <EOL> LOG . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> { '<STR_LIT:id>' : img_id , <EOL> '<STR_LIT>' : base_file } ) <EOL> if os . path . exists ( base_file ) : <EOL> libvirt_utils . update_mtime ( base_file ) <EOL> def _age_and_verify_swap_images ( self , context , base_dir ) : <EOL> LOG . debug ( '<STR_LIT>' ) <EOL> for ent in self . back_swap_images : <EOL> base_file = os . path . join ( base_dir , ent ) <EOL> if ent in self . used_swap_images and os . path . exists ( base_file ) : <EOL> libvirt_utils . update_mtime ( base_file ) <EOL> elif self . remove_unused_base_images : <EOL> self . _remove_swap_file ( base_file ) <EOL> error_images = self . used_swap_images - self . back_swap_images <EOL> for error_image in error_images : <EOL> LOG . warning ( _LW ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , error_image ) <EOL> def _age_and_verify_cached_images ( self , context , all_instances , base_dir ) : <EOL> LOG . debug ( '<STR_LIT>' ) <EOL> for img in self . used_images : <EOL> fingerprint = hashlib . sha1 ( img ) . hexdigest ( ) <EOL> LOG . debug ( '<STR_LIT>' , <EOL> { '<STR_LIT:id>' : img , <EOL> '<STR_LIT>' : fingerprint } ) <EOL> for result in self . _find_base_file ( base_dir , fingerprint ) : <EOL> base_file , image_small , image_resized = result <EOL> self . _handle_base_image ( img , base_file ) <EOL> if not image_small and not image_resized : <EOL> self . originals . append ( base_file ) <EOL> inuse_backing_images = self . _list_backing_images ( ) <EOL> for backing_path in inuse_backing_images : <EOL> if backing_path not in self . active_base_files : <EOL> self . active_base_files . append ( backing_path ) <EOL> for img in self . unexplained_images : <EOL> LOG . warning ( _LW ( '<STR_LIT>' ) , img ) <EOL> self . removable_base_files . append ( img ) <EOL> if self . active_base_files : <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , <EOL> '<STR_LIT:U+0020>' . join ( self . active_base_files ) ) <EOL> if self . corrupt_base_files : <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , <EOL> '<STR_LIT:U+0020>' . join ( self . corrupt_base_files ) ) <EOL> if self . removable_base_files : <EOL> LOG . info ( _LI ( '<STR_LIT>' ) , <EOL> '<STR_LIT:U+0020>' . join ( self . removable_base_files ) ) <EOL> if self . remove_unused_base_images : <EOL> for base_file in self . removable_base_files : <EOL> self . _remove_base_file ( base_file ) <EOL> LOG . debug ( '<STR_LIT>' ) <EOL> def _get_base ( self ) : <EOL> base_dir = os . path . join ( CONF . instances_path , <EOL> CONF . image_cache_subdirectory_name ) <EOL> if not os . path . exists ( base_dir ) : <EOL> LOG . debug ( '<STR_LIT>' , <EOL> base_dir ) <EOL> return <EOL> return base_dir <EOL> def update ( self , context , all_instances ) : <EOL> base_dir = self . _get_base ( ) <EOL> if not base_dir : <EOL> return <EOL> self . _reset_state ( ) <EOL> self . _list_base_images ( base_dir ) <EOL> running = self . _list_running_instances ( context , all_instances ) <EOL> self . used_images = running [ '<STR_LIT>' ] <EOL> self . image_popularity = running [ '<STR_LIT>' ] <EOL> self . instance_names = running [ '<STR_LIT>' ] <EOL> self . used_swap_images = running [ '<STR_LIT>' ] <EOL> self . _age_and_verify_cached_images ( context , all_instances , base_dir ) <EOL> self . _age_and_verify_swap_images ( context , base_dir ) </s>
