<s> import os <EOL> import string <EOL> import textwrap <EOL> import six <EOL> from six . moves . configparser import ConfigParser <EOL> from swift . common . utils import ( <EOL> config_true_value , SWIFT_CONF_FILE , whataremyips , list_from_csv ) <EOL> from swift . common . ring import Ring , RingData <EOL> from swift . common . utils import quorum_size <EOL> from swift . common . exceptions import RingValidationError <EOL> from pyeclib . ec_iface import ECDriver , ECDriverError , VALID_EC_TYPES <EOL> LEGACY_POLICY_NAME = '<STR_LIT>' <EOL> VALID_CHARS = '<STR_LIT:->' + string . ascii_letters + string . digits <EOL> DEFAULT_POLICY_TYPE = REPL_POLICY = '<STR_LIT>' <EOL> EC_POLICY = '<STR_LIT>' <EOL> DEFAULT_EC_OBJECT_SEGMENT_SIZE = <NUM_LIT> <EOL> class BindPortsCache ( object ) : <EOL> def __init__ ( self , swift_dir , bind_ip ) : <EOL> self . swift_dir = swift_dir <EOL> self . mtimes_by_ring_path = { } <EOL> self . portsets_by_ring_path = { } <EOL> self . my_ips = set ( whataremyips ( bind_ip ) ) <EOL> def all_bind_ports_for_node ( self ) : <EOL> """<STR_LIT>""" <EOL> for policy in POLICIES : <EOL> serialized_path = os . path . join ( self . swift_dir , <EOL> policy . ring_name + '<STR_LIT>' ) <EOL> try : <EOL> new_mtime = os . path . getmtime ( serialized_path ) <EOL> except OSError : <EOL> continue <EOL> old_mtime = self . mtimes_by_ring_path . get ( serialized_path ) <EOL> if not old_mtime or old_mtime != new_mtime : <EOL> self . portsets_by_ring_path [ serialized_path ] = set ( <EOL> dev [ '<STR_LIT:port>' ] <EOL> for dev in RingData . load ( serialized_path , <EOL> metadata_only = True ) . devs <EOL> if dev and dev [ '<STR_LIT>' ] in self . my_ips ) <EOL> self . mtimes_by_ring_path [ serialized_path ] = new_mtime <EOL> return six . moves . reduce ( set . union , <EOL> self . portsets_by_ring_path . values ( ) , set ( ) ) <EOL> class PolicyError ( ValueError ) : <EOL> def __init__ ( self , msg , index = None ) : <EOL> if index is not None : <EOL> msg += '<STR_LIT>' % index <EOL> super ( PolicyError , self ) . __init__ ( msg ) <EOL> def _get_policy_string ( base , policy_index ) : <EOL> if policy_index == <NUM_LIT:0> or policy_index is None : <EOL> return_string = base <EOL> else : <EOL> return_string = base + "<STR_LIT>" % int ( policy_index ) <EOL> return return_string <EOL> def get_policy_string ( base , policy_or_index ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( policy_or_index , BaseStoragePolicy ) : <EOL> policy = policy_or_index <EOL> else : <EOL> policy = POLICIES . get_by_index ( policy_or_index ) <EOL> if policy is None : <EOL> raise PolicyError ( "<STR_LIT>" , index = policy_or_index ) <EOL> return _get_policy_string ( base , int ( policy ) ) <EOL> def split_policy_string ( policy_string ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT:->' in policy_string : <EOL> base , policy_index = policy_string . rsplit ( '<STR_LIT:->' , <NUM_LIT:1> ) <EOL> else : <EOL> base , policy_index = policy_string , None <EOL> policy = POLICIES . get_by_index ( policy_index ) <EOL> if get_policy_string ( base , policy ) != policy_string : <EOL> raise PolicyError ( "<STR_LIT>" , index = policy_index ) <EOL> return base , policy <EOL> class BaseStoragePolicy ( object ) : <EOL> """<STR_LIT>""" <EOL> policy_type_to_policy_cls = { } <EOL> def __init__ ( self , idx , name = '<STR_LIT>' , is_default = False , is_deprecated = False , <EOL> object_ring = None , aliases = '<STR_LIT>' ) : <EOL> if type ( self ) == BaseStoragePolicy : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> try : <EOL> self . idx = int ( idx ) <EOL> except ValueError : <EOL> raise PolicyError ( '<STR_LIT>' , idx ) <EOL> if self . idx < <NUM_LIT:0> : <EOL> raise PolicyError ( '<STR_LIT>' , idx ) <EOL> self . alias_list = [ ] <EOL> self . add_name ( name ) <EOL> if aliases : <EOL> names_list = list_from_csv ( aliases ) <EOL> for alias in names_list : <EOL> if alias == name : <EOL> continue <EOL> self . add_name ( alias ) <EOL> self . is_deprecated = config_true_value ( is_deprecated ) <EOL> self . is_default = config_true_value ( is_default ) <EOL> if self . policy_type not in BaseStoragePolicy . policy_type_to_policy_cls : <EOL> raise PolicyError ( '<STR_LIT>' , self . policy_type ) <EOL> if self . is_deprecated and self . is_default : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> '<STR_LIT>' , self . idx ) <EOL> self . ring_name = _get_policy_string ( '<STR_LIT:object>' , self . idx ) <EOL> self . object_ring = object_ring <EOL> @ property <EOL> def name ( self ) : <EOL> return self . alias_list [ <NUM_LIT:0> ] <EOL> @ name . setter <EOL> def name_setter ( self , name ) : <EOL> self . _validate_policy_name ( name ) <EOL> self . alias_list [ <NUM_LIT:0> ] = name <EOL> @ property <EOL> def aliases ( self ) : <EOL> return "<STR_LIT:U+002CU+0020>" . join ( self . alias_list ) <EOL> def __int__ ( self ) : <EOL> return self . idx <EOL> def __cmp__ ( self , other ) : <EOL> return cmp ( self . idx , int ( other ) ) <EOL> def __repr__ ( self ) : <EOL> return ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % ( self . __class__ . __name__ , self . idx , self . alias_list , <EOL> self . is_default , self . is_deprecated , self . policy_type ) <EOL> @ classmethod <EOL> def register ( cls , policy_type ) : <EOL> """<STR_LIT>""" <EOL> def register_wrapper ( policy_cls ) : <EOL> if policy_type in cls . policy_type_to_policy_cls : <EOL> raise PolicyError ( <EOL> '<STR_LIT>' % ( <EOL> cls . policy_type_to_policy_cls [ policy_type ] , <EOL> policy_type ) ) <EOL> cls . policy_type_to_policy_cls [ policy_type ] = policy_cls <EOL> policy_cls . policy_type = policy_type <EOL> return policy_cls <EOL> return register_wrapper <EOL> @ classmethod <EOL> def _config_options_map ( cls ) : <EOL> """<STR_LIT>""" <EOL> return { <EOL> '<STR_LIT:name>' : '<STR_LIT:name>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:default>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> @ classmethod <EOL> def from_config ( cls , policy_index , options ) : <EOL> config_to_policy_option_map = cls . _config_options_map ( ) <EOL> policy_options = { } <EOL> for config_option , value in options . items ( ) : <EOL> try : <EOL> policy_option = config_to_policy_option_map [ config_option ] <EOL> except KeyError : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> '<STR_LIT>' % config_option , <EOL> index = policy_index ) <EOL> policy_options [ policy_option ] = value <EOL> return cls ( policy_index , ** policy_options ) <EOL> def get_info ( self , config = False ) : <EOL> """<STR_LIT>""" <EOL> info = { } <EOL> for config_option , policy_attribute in self . _config_options_map ( ) . items ( ) : <EOL> info [ config_option ] = getattr ( self , policy_attribute ) <EOL> if not config : <EOL> if not self . is_default : <EOL> info . pop ( '<STR_LIT:default>' ) <EOL> if not self . is_deprecated : <EOL> info . pop ( '<STR_LIT>' ) <EOL> info . pop ( '<STR_LIT>' ) <EOL> return info <EOL> def _validate_policy_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> if not name : <EOL> raise PolicyError ( '<STR_LIT>' % name , self . idx ) <EOL> if not all ( c in VALID_CHARS for c in name ) : <EOL> msg = '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' % ( VALID_CHARS , name ) <EOL> raise PolicyError ( msg , self . idx ) <EOL> if name . upper ( ) == LEGACY_POLICY_NAME . upper ( ) and self . idx != <NUM_LIT:0> : <EOL> msg = '<STR_LIT>' '<STR_LIT>' % ( LEGACY_POLICY_NAME , name ) <EOL> raise PolicyError ( msg , self . idx ) <EOL> if name . upper ( ) in ( existing_name . upper ( ) for existing_name <EOL> in self . alias_list ) : <EOL> msg = '<STR_LIT>' % name <EOL> raise PolicyError ( msg , self . idx ) <EOL> def add_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> self . _validate_policy_name ( name ) <EOL> self . alias_list . append ( name ) <EOL> def remove_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> if name not in self . alias_list : <EOL> raise PolicyError ( "<STR_LIT>" <EOL> % ( name , self . idx ) ) <EOL> if len ( self . alias_list ) == <NUM_LIT:1> : <EOL> raise PolicyError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( name , self . idx ) ) <EOL> else : <EOL> self . alias_list . remove ( name ) <EOL> def change_primary_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> if name == self . name : <EOL> return <EOL> elif name in self . alias_list : <EOL> self . remove_name ( name ) <EOL> else : <EOL> self . _validate_policy_name ( name ) <EOL> self . alias_list . insert ( <NUM_LIT:0> , name ) <EOL> def _validate_ring ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def load_ring ( self , swift_dir ) : <EOL> """<STR_LIT>""" <EOL> if self . object_ring : <EOL> return <EOL> self . object_ring = Ring ( swift_dir , ring_name = self . ring_name ) <EOL> self . _validate_ring ( ) <EOL> @ property <EOL> def quorum ( self ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( ) <EOL> @ BaseStoragePolicy . register ( REPL_POLICY ) <EOL> class StoragePolicy ( BaseStoragePolicy ) : <EOL> """<STR_LIT>""" <EOL> @ property <EOL> def quorum ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . object_ring : <EOL> raise PolicyError ( '<STR_LIT>' ) <EOL> return quorum_size ( self . object_ring . replica_count ) <EOL> @ BaseStoragePolicy . register ( EC_POLICY ) <EOL> class ECStoragePolicy ( BaseStoragePolicy ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , idx , name = '<STR_LIT>' , aliases = '<STR_LIT>' , is_default = False , <EOL> is_deprecated = False , object_ring = None , <EOL> ec_segment_size = DEFAULT_EC_OBJECT_SEGMENT_SIZE , <EOL> ec_type = None , ec_ndata = None , ec_nparity = None ) : <EOL> super ( ECStoragePolicy , self ) . __init__ ( <EOL> idx = idx , name = name , aliases = aliases , is_default = is_default , <EOL> is_deprecated = is_deprecated , object_ring = object_ring ) <EOL> if ec_type is None : <EOL> raise PolicyError ( '<STR_LIT>' ) <EOL> if ec_type not in VALID_EC_TYPES : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( ec_type , self . name , <EOL> '<STR_LIT:U+002CU+0020>' . join ( VALID_EC_TYPES ) ) ) <EOL> self . _ec_type = ec_type <EOL> try : <EOL> value = int ( ec_ndata ) <EOL> if value <= <NUM_LIT:0> : <EOL> raise ValueError <EOL> self . _ec_ndata = value <EOL> except ( TypeError , ValueError ) : <EOL> raise PolicyError ( '<STR_LIT>' % <EOL> ec_ndata , index = self . idx ) <EOL> try : <EOL> value = int ( ec_nparity ) <EOL> if value <= <NUM_LIT:0> : <EOL> raise ValueError <EOL> self . _ec_nparity = value <EOL> except ( TypeError , ValueError ) : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> % ec_nparity , index = self . idx ) <EOL> try : <EOL> value = int ( ec_segment_size ) <EOL> if value <= <NUM_LIT:0> : <EOL> raise ValueError <EOL> self . _ec_segment_size = value <EOL> except ( TypeError , ValueError ) : <EOL> raise PolicyError ( '<STR_LIT>' % <EOL> ec_segment_size , index = self . idx ) <EOL> try : <EOL> self . pyeclib_driver = ECDriver ( k = self . _ec_ndata , m = self . _ec_nparity , <EOL> ec_type = self . _ec_type ) <EOL> except ECDriverError as e : <EOL> raise PolicyError ( "<STR_LIT>" % e , <EOL> index = self . idx ) <EOL> self . _ec_quorum_size = self . _ec_ndata + self . pyeclib_driver . min_parity_fragments_needed ( ) <EOL> @ property <EOL> def ec_type ( self ) : <EOL> return self . _ec_type <EOL> @ property <EOL> def ec_ndata ( self ) : <EOL> return self . _ec_ndata <EOL> @ property <EOL> def ec_nparity ( self ) : <EOL> return self . _ec_nparity <EOL> @ property <EOL> def ec_segment_size ( self ) : <EOL> return self . _ec_segment_size <EOL> @ property <EOL> def fragment_size ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . pyeclib_driver . get_segment_info ( <EOL> self . ec_segment_size , self . ec_segment_size ) [ '<STR_LIT>' ] <EOL> @ property <EOL> def ec_scheme_description ( self ) : <EOL> """<STR_LIT>""" <EOL> return "<STR_LIT>" % ( self . _ec_type , self . _ec_ndata , self . _ec_nparity ) <EOL> def __repr__ ( self ) : <EOL> return ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % ( super ( ECStoragePolicy , self ) . __repr__ ( ) , self . ec_type , <EOL> self . ec_segment_size , self . ec_ndata , self . ec_nparity ) <EOL> @ classmethod <EOL> def _config_options_map ( cls ) : <EOL> options = super ( ECStoragePolicy , cls ) . _config_options_map ( ) <EOL> options . update ( { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } ) <EOL> return options <EOL> def get_info ( self , config = False ) : <EOL> info = super ( ECStoragePolicy , self ) . get_info ( config = config ) <EOL> if not config : <EOL> info . pop ( '<STR_LIT>' ) <EOL> info . pop ( '<STR_LIT>' ) <EOL> info . pop ( '<STR_LIT>' ) <EOL> info . pop ( '<STR_LIT>' ) <EOL> return info <EOL> def _validate_ring ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . object_ring : <EOL> raise PolicyError ( '<STR_LIT>' ) <EOL> nodes_configured = self . object_ring . replica_count <EOL> if nodes_configured != ( self . ec_ndata + self . ec_nparity ) : <EOL> raise RingValidationError ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( <EOL> self . name , self . ec_ndata + self . ec_nparity , <EOL> nodes_configured ) ) <EOL> @ property <EOL> def quorum ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _ec_quorum_size <EOL> class StoragePolicyCollection ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , pols ) : <EOL> self . default = [ ] <EOL> self . by_name = { } <EOL> self . by_index = { } <EOL> self . _validate_policies ( pols ) <EOL> def _add_policy ( self , policy ) : <EOL> """<STR_LIT>""" <EOL> for name in policy . alias_list : <EOL> self . by_name [ name . upper ( ) ] = policy <EOL> self . by_index [ int ( policy ) ] = policy <EOL> def __repr__ ( self ) : <EOL> return ( textwrap . dedent ( """<STR_LIT>""" ) % '<STR_LIT>' . join ( repr ( p ) for p in self ) ) . strip ( ) <EOL> def __len__ ( self ) : <EOL> return len ( self . by_index ) <EOL> def __getitem__ ( self , key ) : <EOL> return self . by_index [ key ] <EOL> def __iter__ ( self ) : <EOL> return iter ( self . by_index . values ( ) ) <EOL> def _validate_policies ( self , policies ) : <EOL> """<STR_LIT>""" <EOL> for policy in policies : <EOL> if int ( policy ) in self . by_index : <EOL> raise PolicyError ( '<STR_LIT>' % ( <EOL> policy , self . get_by_index ( int ( policy ) ) ) ) <EOL> for name in policy . alias_list : <EOL> if name . upper ( ) in self . by_name : <EOL> raise PolicyError ( '<STR_LIT>' % ( <EOL> policy , self . get_by_name ( name ) ) ) <EOL> if policy . is_default : <EOL> if not self . default : <EOL> self . default = policy <EOL> else : <EOL> raise PolicyError ( <EOL> '<STR_LIT>' % ( <EOL> policy , self . default ) ) <EOL> self . _add_policy ( policy ) <EOL> if <NUM_LIT:0> not in self . by_index : <EOL> if len ( self ) != <NUM_LIT:0> : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> self . _add_policy ( StoragePolicy ( <NUM_LIT:0> , name = LEGACY_POLICY_NAME ) ) <EOL> enabled_policies = [ p for p in self if not p . is_deprecated ] <EOL> if not enabled_policies : <EOL> raise PolicyError ( "<STR_LIT>" ) <EOL> if not self . default : <EOL> if len ( self ) > <NUM_LIT:1> : <EOL> raise PolicyError ( "<STR_LIT>" ) <EOL> self . default = self [ <NUM_LIT:0> ] <EOL> self . default . is_default = True <EOL> def get_by_name ( self , name ) : <EOL> """<STR_LIT>""" <EOL> return self . by_name . get ( name . upper ( ) ) <EOL> def get_by_index ( self , index ) : <EOL> """<STR_LIT>""" <EOL> if index in ( '<STR_LIT>' , None ) : <EOL> index = <NUM_LIT:0> <EOL> else : <EOL> try : <EOL> index = int ( index ) <EOL> except ValueError : <EOL> return None <EOL> return self . by_index . get ( index ) <EOL> @ property <EOL> def legacy ( self ) : <EOL> return self . get_by_index ( None ) <EOL> def get_object_ring ( self , policy_idx , swift_dir ) : <EOL> """<STR_LIT>""" <EOL> policy = self . get_by_index ( policy_idx ) <EOL> if not policy : <EOL> raise PolicyError ( "<STR_LIT>" % policy_idx ) <EOL> if not policy . object_ring : <EOL> policy . load_ring ( swift_dir ) <EOL> return policy . object_ring <EOL> def get_policy_info ( self ) : <EOL> """<STR_LIT>""" <EOL> policy_info = [ ] <EOL> for pol in self : <EOL> if pol . is_deprecated : <EOL> continue <EOL> policy_entry = pol . get_info ( ) <EOL> policy_info . append ( policy_entry ) <EOL> return policy_info <EOL> def add_policy_alias ( self , policy_index , * aliases ) : <EOL> """<STR_LIT>""" <EOL> policy = self . get_by_index ( policy_index ) <EOL> for alias in aliases : <EOL> if alias . upper ( ) in self . by_name : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( alias , <EOL> self . get_by_name ( alias ) ) ) <EOL> else : <EOL> policy . add_name ( alias ) <EOL> self . by_name [ alias . upper ( ) ] = policy <EOL> def remove_policy_alias ( self , * aliases ) : <EOL> """<STR_LIT>""" <EOL> for alias in aliases : <EOL> policy = self . get_by_name ( alias ) <EOL> if not policy : <EOL> raise PolicyError ( '<STR_LIT>' % alias ) <EOL> if len ( policy . alias_list ) == <NUM_LIT:1> : <EOL> raise PolicyError ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( <EOL> policy , alias ) ) <EOL> else : <EOL> policy . remove_name ( alias ) <EOL> del self . by_name [ alias . upper ( ) ] <EOL> def change_policy_primary_name ( self , policy_index , new_name ) : <EOL> """<STR_LIT>""" <EOL> policy = self . get_by_index ( policy_index ) <EOL> name_taken = self . get_by_name ( new_name ) <EOL> if name_taken and name_taken != policy : <EOL> raise PolicyError ( '<STR_LIT>' % <EOL> ( self . get_by_name ( new_name ) . idx , new_name ) ) <EOL> else : <EOL> policy . change_primary_name ( new_name ) <EOL> self . by_name [ new_name . upper ( ) ] = policy <EOL> def parse_storage_policies ( conf ) : <EOL> """<STR_LIT>""" <EOL> policies = [ ] <EOL> for section in conf . sections ( ) : <EOL> if not section . startswith ( '<STR_LIT>' ) : <EOL> continue <EOL> policy_index = section . split ( '<STR_LIT::>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> config_options = dict ( conf . items ( section ) ) <EOL> policy_type = config_options . pop ( '<STR_LIT>' , DEFAULT_POLICY_TYPE ) <EOL> policy_cls = BaseStoragePolicy . policy_type_to_policy_cls [ policy_type ] <EOL> policy = policy_cls . from_config ( policy_index , config_options ) <EOL> policies . append ( policy ) <EOL> return StoragePolicyCollection ( policies ) <EOL> class StoragePolicySingleton ( object ) : <EOL> """<STR_LIT>""" <EOL> def __iter__ ( self ) : <EOL> return iter ( _POLICIES ) <EOL> def __len__ ( self ) : <EOL> return len ( _POLICIES ) <EOL> def __getitem__ ( self , key ) : <EOL> return _POLICIES [ key ] <EOL> def __getattribute__ ( self , name ) : <EOL> return getattr ( _POLICIES , name ) <EOL> def __repr__ ( self ) : <EOL> return repr ( _POLICIES ) <EOL> def reload_storage_policies ( ) : <EOL> """<STR_LIT>""" <EOL> global _POLICIES <EOL> policy_conf = ConfigParser ( ) <EOL> policy_conf . read ( SWIFT_CONF_FILE ) <EOL> try : <EOL> _POLICIES = parse_storage_policies ( policy_conf ) <EOL> except PolicyError as e : <EOL> raise SystemExit ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( SWIFT_CONF_FILE , e ) ) <EOL> _POLICIES = None <EOL> reload_storage_policies ( ) <EOL> POLICIES = StoragePolicySingleton ( ) </s>
