<s> """<STR_LIT>""" <EOL> import os , re <EOL> import whisper <EOL> from os . path import join , exists <EOL> from carbon . conf import OrderedConfigParser , settings <EOL> from carbon . util import pickle <EOL> from carbon import log <EOL> STORAGE_SCHEMAS_CONFIG = join ( settings . CONF_DIR , '<STR_LIT>' ) <EOL> STORAGE_AGGREGATION_CONFIG = join ( settings . CONF_DIR , '<STR_LIT>' ) <EOL> STORAGE_LISTS_DIR = join ( settings . CONF_DIR , '<STR_LIT>' ) <EOL> def getFilesystemPath ( metric ) : <EOL> return join ( settings . LOCAL_DATA_DIR , metric . replace ( '<STR_LIT:.>' , '<STR_LIT:/>' ) ) + '<STR_LIT>' <EOL> class Schema : <EOL> def test ( self , metric ) : <EOL> raise NotImplementedError ( ) <EOL> def matches ( self , metric ) : <EOL> return bool ( self . test ( metric ) ) <EOL> class DefaultSchema ( Schema ) : <EOL> def __init__ ( self , name , archives ) : <EOL> self . name = name <EOL> self . archives = archives <EOL> def test ( self , metric ) : <EOL> return True <EOL> class PatternSchema ( Schema ) : <EOL> def __init__ ( self , name , pattern , archives ) : <EOL> self . name = name <EOL> self . pattern = pattern <EOL> self . regex = re . compile ( pattern ) <EOL> self . archives = archives <EOL> def test ( self , metric ) : <EOL> return self . regex . search ( metric ) <EOL> class ListSchema ( Schema ) : <EOL> def __init__ ( self , name , listName , archives ) : <EOL> self . name = name <EOL> self . listName = listName <EOL> self . archives = archives <EOL> self . path = join ( settings . WHITELISTS_DIR , listName ) <EOL> if exists ( self . path ) : <EOL> self . mtime = os . stat ( self . path ) . st_mtime <EOL> fh = open ( self . path , '<STR_LIT:rb>' ) <EOL> self . members = pickle . load ( fh ) <EOL> fh . close ( ) <EOL> else : <EOL> self . mtime = <NUM_LIT:0> <EOL> self . members = frozenset ( ) <EOL> def test ( self , metric ) : <EOL> if exists ( self . path ) : <EOL> current_mtime = os . stat ( self . path ) . st_mtime <EOL> if current_mtime > self . mtime : <EOL> self . mtime = current_mtime <EOL> fh = open ( self . path , '<STR_LIT:rb>' ) <EOL> self . members = pickle . load ( fh ) <EOL> fh . close ( ) <EOL> return metric in self . members <EOL> class Archive : <EOL> def __init__ ( self , secondsPerPoint , points ) : <EOL> self . secondsPerPoint = int ( secondsPerPoint ) <EOL> self . points = int ( points ) <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" % ( self . secondsPerPoint , self . points ) <EOL> def getTuple ( self ) : <EOL> return ( self . secondsPerPoint , self . points ) <EOL> @ staticmethod <EOL> def fromString ( retentionDef ) : <EOL> ( secondsPerPoint , points ) = whisper . parseRetentionDef ( retentionDef ) <EOL> return Archive ( secondsPerPoint , points ) <EOL> def loadStorageSchemas ( ) : <EOL> schemaList = [ ] <EOL> config = OrderedConfigParser ( ) <EOL> config . read ( STORAGE_SCHEMAS_CONFIG ) <EOL> for section in config . sections ( ) : <EOL> options = dict ( config . items ( section ) ) <EOL> matchAll = options . get ( '<STR_LIT>' ) <EOL> pattern = options . get ( '<STR_LIT>' ) <EOL> listName = options . get ( '<STR_LIT:list>' ) <EOL> retentions = options [ '<STR_LIT>' ] . split ( '<STR_LIT:U+002C>' ) <EOL> archives = [ Archive . fromString ( s ) for s in retentions ] <EOL> if matchAll : <EOL> mySchema = DefaultSchema ( section , archives ) <EOL> elif pattern : <EOL> mySchema = PatternSchema ( section , pattern , archives ) <EOL> elif listName : <EOL> mySchema = ListSchema ( section , listName , archives ) <EOL> archiveList = [ a . getTuple ( ) for a in archives ] <EOL> try : <EOL> whisper . validateArchiveList ( archiveList ) <EOL> schemaList . append ( mySchema ) <EOL> except InvalidConfiguration , e : <EOL> log . msg ( "<STR_LIT>" % ( section , e . message ) ) <EOL> schemaList . append ( defaultSchema ) <EOL> return schemaList <EOL> def loadAggregationSchemas ( ) : <EOL> schemaList = [ ] <EOL> config = OrderedConfigParser ( ) <EOL> try : <EOL> config . read ( STORAGE_AGGREGATION_CONFIG ) <EOL> except IOError : <EOL> log . msg ( "<STR_LIT>" % STORAGE_AGGREGATION_CONFIG ) <EOL> for section in config . sections ( ) : <EOL> options = dict ( config . items ( section ) ) <EOL> matchAll = options . get ( '<STR_LIT>' ) <EOL> pattern = options . get ( '<STR_LIT>' ) <EOL> listName = options . get ( '<STR_LIT:list>' ) <EOL> xFilesFactor = options . get ( '<STR_LIT>' ) <EOL> aggregationMethod = options . get ( '<STR_LIT>' ) <EOL> try : <EOL> if xFilesFactor is not None : <EOL> xFilesFactor = float ( xFilesFactor ) <EOL> assert <NUM_LIT:0> <= xFilesFactor <= <NUM_LIT:1> <EOL> if aggregationMethod is not None : <EOL> assert aggregationMethod in whisper . aggregationMethods <EOL> except : <EOL> log . msg ( "<STR_LIT>" % section ) <EOL> continue <EOL> archives = ( xFilesFactor , aggregationMethod ) <EOL> if matchAll : <EOL> mySchema = DefaultSchema ( section , archives ) <EOL> elif pattern : <EOL> mySchema = PatternSchema ( section , pattern , archives ) <EOL> elif listName : <EOL> mySchema = ListSchema ( section , listName , archives ) <EOL> schemaList . append ( mySchema ) <EOL> schemaList . append ( defaultAggregation ) <EOL> return schemaList <EOL> defaultArchive = Archive ( <NUM_LIT> , <NUM_LIT> * <NUM_LIT> * <NUM_LIT:7> ) <EOL> defaultSchema = DefaultSchema ( '<STR_LIT:default>' , [ defaultArchive ] ) <EOL> defaultAggregation = DefaultSchema ( '<STR_LIT:default>' , ( None , None ) ) </s>
