original_docstring: 
          - Create a Redis set of all Ionosphere enabled metrics.
          - Manage the ionosphere.untrainable_metrics set, removing items when
            they 'expire'
  
          :param i: python process id
          :return: returns True
  
          
docstring: Create a Redis set of all Ionosphere enabled metrics.
  Manage the ionosphere.untrainable_metrics set, removing items when
  they 'expire'
docstring_tokens: Create a Redis set of all Ionosphere enabled metrics . Manage the ionosphere . untrainable_metrics set removing items when they ' expire '
code: def manage_ionosphere_unique_metrics(self):
          """
          - Create a Redis set of all Ionosphere enabled metrics.
          - Manage the ionosphere.untrainable_metrics set, removing items when
            they 'expire'
  
          :param i: python process id
          :return: returns True
  
          """
  
          def get_an_engine():
              try:
                  engine, log_msg, trace = get_engine(skyline_app)
                  return engine, log_msg, trace
              except:
                  trace = traceback.format_exc()
                  logger.error('%s' % trace)
                  log_msg = 'error :: failed to get MySQL engine for manage_ionosphere_unique_metrics'
                  logger.error('%s' % log_msg)
                  return None, log_msg, trace
  
          ionosphere_unique_metrics_count = 0
          redis_ionosphere_unique_metrics = None
          ionosphere_unique_metrics = []
          try:
              # ionosphere_unique_metrics = list(self.redis_conn.smembers('ionosphere.unique_metrics'))
              # @modified 20191030 - Bug #3266: py3 Redis binary objects not strings
              #                      Branch #3262: py3
              # redis_ionosphere_unique_metrics = self.redis_conn.smembers('ionosphere.unique_metrics')
              redis_ionosphere_unique_metrics = self.redis_conn_decoded.smembers('ionosphere.unique_metrics')
          except:
              logger.error(traceback.format_exc())
              logger.error('error :: failed to get ionosphere.unique_metrics from Redis')
              # ionosphere_unique_metrics = []
  
          if redis_ionosphere_unique_metrics is not None:
              ionosphere_unique_metrics = list(redis_ionosphere_unique_metrics)
              ionosphere_unique_metrics_count = len(ionosphere_unique_metrics)
              logger.info('Redis ionosphere.unique_metrics set has %s metrics' % (str(ionosphere_unique_metrics_count)))
          else:
              logger.info('Redis ionosphere.unique_metrics unknown setting to []')
              ionosphere_unique_metrics = []
  
          manage_ionosphere_unique_metrics = True
          manage_ionosphere_unique_metrics_key = []
          try:
              manage_ionosphere_unique_metrics_key = self.redis_conn.get('ionosphere.manage_ionosphere_unique_metrics')
          except Exception as e:
              if LOCAL_DEBUG:
                  logger.error('error :: could not query Redis for ionosphere.manage_ionosphere_unique_metrics key: %s' % str(e))
  
          if manage_ionosphere_unique_metrics_key is not None:
              manage_ionosphere_unique_metrics = False
  
          logger.info('getting MySQL engine for ionosphere_enabled_metrics')
          try:
              engine, log_msg, trace = get_an_engine()
              logger.info(log_msg)
          except:
              logger.error(traceback.format_exc())
              logger.error('error :: could not get a MySQL engine for ionosphere_enabled_metrics')
              return False
          if not engine:
              logger.error('error :: MySQL engine not obtained for ionosphere_enabled_metrics')
              return False
  
          # Determine the metrics that have ionosphere_enabled
          # @added 20170103 - Task #1658: Patterning Skyline Ionosphere
          # TODO: We need 2 sets not just ionosphere.unique_metrics otherwise
          #       if a metric is switch from Analyzer to Mirage will send all
          #       matched anomalies to Ionosphere even if there is no features
          #       profile at the specified duration.
          # ionosphere.analyzer.unique_metrics (at FULL_DURATION)
          # ionosphere.mirage.unique_metrics (NOT at FULL_DURATION)
          # @modified 20170108 - Feature #1852: Ionosphere - features_profile matched graphite graphs
          # Yes those ^^ are needed, MySQL join?
          ionosphere_enabled_metrics = []
          ionosphere_metrics_count = 0
          query_ok = False
          try:
              stmt = 'select metric from metrics where ionosphere_enabled=1'
              connection = engine.connect()
              for row in engine.execute(stmt):
                  metric_basename = row['metric']
                  metric_name = '%s%s' % (str(settings.FULL_NAMESPACE), str(metric_basename))
                  ionosphere_enabled_metrics.append(metric_name)
              connection.close()
              query_ok = True
          except:
              logger.error(traceback.format_exc())
              logger.error('error :: could not determine ionosphere_enabled metrics from the DB to manage ionosphere.unique_metrics Redis set')
  
          ionosphere_metrics_count = len(ionosphere_enabled_metrics)
          logger.info('db has %s ionosphere_enabled metrics' % (str(ionosphere_metrics_count)))
  
          # @added 20190528 - Branch #3002: docker
          if ionosphere_metrics_count == 0:
              ionosphere_enabled_metrics = ['none']
  
          if manage_ionosphere_unique_metrics:
              # Testing the query was fine and Ionosphere metrics can go to 0 if
              # all were disabled
              if query_ok:
                  manage_ionosphere_unique_metrics = True
              else:
                  manage_ionosphere_unique_metrics = False
  
          if manage_ionosphere_unique_metrics:
              for metric_name in ionosphere_enabled_metrics:
                  try:
                      self.redis_conn.sadd('ionosphere.new_unique_metrics', metric_name)
                      # logger.info('added %s to ionosphere.new_unique_metrics Redis set' % metric_name)
                  except:
                      logger.error(traceback.format_exc())
                      logger.info('error :: failed to add %s to ionosphere.new_unique_metrics Redis set' % metric_name)
                  try:
                      self.redis_conn.setex('ionosphere.manage_ionosphere_unique_metrics', 300, time())
                  except:
                      logger.error('error :: failed to set key :: ionosphere.manage_ionosphere_unique_metrics')
              try:
                  logger.info('replacing Redis ionosphere.unique_metrics via rename of ionosphere.new_unique_metrics')
                  self.redis_conn.rename('ionosphere.new_unique_metrics', 'ionosphere.unique_metrics')
                  manage_ionosphere_unique_metrics = False
                  ionosphere_unique_metrics = []
              except Exception as e:
                  if str(e) == 'no such key':
                      logger.info('could not rename Redis set ionosphere.new_unique_metrics to ionosphere.unique_metrics: %s' % str(e))
                  else:
                      logger.error('error :: could not rename Redis set ionosphere.new_unique_metrics to ionosphere.unique_metrics: %s' % str(e))
              try:
                  self.redis_conn.setex('ionosphere.manage_ionosphere_unique_metrics', 300, time())
              except:
                  logger.error('error :: failed to set key :: ionosphere.manage_ionosphere_unique_metrics')
              redis_ionosphere_unique_metrics = []
              try:
                  # ionosphere_unique_metrics = list(self.redis_conn.smembers('ionosphere.unique_metrics'))
                  # @modified 20191030 - Bug #3266: py3 Redis binary objects not strings
                  #                      Branch #3262: py3
                  # redis_ionosphere_unique_metrics = self.redis_conn.smembers('ionosphere.unique_metrics')
                  redis_ionosphere_unique_metrics = self.redis_conn_decoded.smembers('ionosphere.unique_metrics')
              except:
                  logger.error(traceback.format_exc())
                  logger.error('error :: failed to get ionosphere.unique_metrics from Redis')
                  # ionosphere_unique_metrics = []
  
              if redis_ionosphere_unique_metrics is not None:
                  ionosphere_unique_metrics = list(redis_ionosphere_unique_metrics)
                  ionosphere_unique_metrics_count = len(ionosphere_unique_metrics)
                  logger.info('the new Redis ionosphere.unique_metrics set has %s metrics' % (str(ionosphere_unique_metrics_count)))
              else:
                  logger.info('Redis ionosphere.unique_metrics unknown setting to []')
                  ionosphere_unique_metrics = []
  
          if engine:
              try:
                  engine.dispose()
              except:
                  logger.error(traceback.format_exc())
                  logger.error('error :: calling engine.dispose()')
  
          # @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
          # Manage ionosphere_untrainable_metrics Redis set
          ionosphere_untrainable_metrics = []
          ionosphere_untrainable_metrics_redis_set = 'ionosphere.untrainable_metrics'
          try:
              ionosphere_untrainable_metrics = list(self.redis_conn_decoded.smembers(ionosphere_untrainable_metrics_redis_set))
          except:
              logger.error(traceback.format_exc())
              logger.error('error :: could not get the ionosphere.untrainable_metrics set from Redis')
          if ionosphere_untrainable_metrics:
              ionosphere_untrainable_metrics_check_time = int(time())
              for ionosphere_untrainable_metric_str in ionosphere_untrainable_metrics:
                  try:
                      ionosphere_untrainable_metric = literal_eval(ionosphere_untrainable_metric_str)
                      ium_remove_after_timestamp = int(ionosphere_untrainable_metric[6])
                      if ionosphere_untrainable_metrics_check_time >= ium_remove_after_timestamp:
                          try:
                              self.redis_conn.srem(ionosphere_untrainable_metrics_redis_set, str(ionosphere_untrainable_metric))
                              logger.info('removed item - %s - from Redis set - %s' % (str(ionosphere_untrainable_metric), ionosphere_untrainable_metrics_redis_set))
                          except:
                              logger.error(traceback.format_exc())
                              logger.error('error :: failed to remove item list from Redis set - %s' % ionosphere_untrainable_metrics_redis_set)
                  except:
                      logger.error(traceback.format_exc())
                      logger.error('error :: failed to manage Redis set %s' % ionosphere_untrainable_metrics_redis_set)
  
          return True
code_tokens: def manage_ionosphere_unique_metrics ( self ) : def get_an_engine ( ) : try : engine , log_msg , trace = get_engine ( skyline_app ) return engine , log_msg , trace except : trace = traceback . format_exc ( ) logger . error ( '%s' % trace ) log_msg = 'error :: failed to get MySQL engine for manage_ionosphere_unique_metrics' logger . error ( '%s' % log_msg ) return None , log_msg , trace ionosphere_unique_metrics_count = 0 redis_ionosphere_unique_metrics = None ionosphere_unique_metrics = [ ] try : redis_ionosphere_unique_metrics = self . redis_conn_decoded . smembers ( 'ionosphere.unique_metrics' ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: failed to get ionosphere.unique_metrics from Redis' ) if redis_ionosphere_unique_metrics is not None : ionosphere_unique_metrics = list ( redis_ionosphere_unique_metrics ) ionosphere_unique_metrics_count = len ( ionosphere_unique_metrics ) logger . info ( 'Redis ionosphere.unique_metrics set has %s metrics' % ( str ( ionosphere_unique_metrics_count ) ) ) else : logger . info ( 'Redis ionosphere.unique_metrics unknown setting to []' ) ionosphere_unique_metrics = [ ] manage_ionosphere_unique_metrics = True manage_ionosphere_unique_metrics_key = [ ] try : manage_ionosphere_unique_metrics_key = self . redis_conn . get ( 'ionosphere.manage_ionosphere_unique_metrics' ) except Exception as e : if LOCAL_DEBUG : logger . error ( 'error :: could not query Redis for ionosphere.manage_ionosphere_unique_metrics key: %s' % str ( e ) ) if manage_ionosphere_unique_metrics_key is not None : manage_ionosphere_unique_metrics = False logger . info ( 'getting MySQL engine for ionosphere_enabled_metrics' ) try : engine , log_msg , trace = get_an_engine ( ) logger . info ( log_msg ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: could not get a MySQL engine for ionosphere_enabled_metrics' ) return False if not engine : logger . error ( 'error :: MySQL engine not obtained for ionosphere_enabled_metrics' ) return False ionosphere_enabled_metrics = [ ] ionosphere_metrics_count = 0 query_ok = False try : stmt = 'select metric from metrics where ionosphere_enabled=1' connection = engine . connect ( ) for row in engine . execute ( stmt ) : metric_basename = row [ 'metric' ] metric_name = '%s%s' % ( str ( settings . FULL_NAMESPACE ) , str ( metric_basename ) ) ionosphere_enabled_metrics . append ( metric_name ) connection . close ( ) query_ok = True except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: could not determine ionosphere_enabled metrics from the DB to manage ionosphere.unique_metrics Redis set' ) ionosphere_metrics_count = len ( ionosphere_enabled_metrics ) logger . info ( 'db has %s ionosphere_enabled metrics' % ( str ( ionosphere_metrics_count ) ) ) if ionosphere_metrics_count == 0 : ionosphere_enabled_metrics = [ 'none' ] if manage_ionosphere_unique_metrics : if query_ok : manage_ionosphere_unique_metrics = True else : manage_ionosphere_unique_metrics = False if manage_ionosphere_unique_metrics : for metric_name in ionosphere_enabled_metrics : try : self . redis_conn . sadd ( 'ionosphere.new_unique_metrics' , metric_name ) except : logger . error ( traceback . format_exc ( ) ) logger . info ( 'error :: failed to add %s to ionosphere.new_unique_metrics Redis set' % metric_name ) try : self . redis_conn . setex ( 'ionosphere.manage_ionosphere_unique_metrics' , 300 , time ( ) ) except : logger . error ( 'error :: failed to set key :: ionosphere.manage_ionosphere_unique_metrics' ) try : logger . info ( 'replacing Redis ionosphere.unique_metrics via rename of ionosphere.new_unique_metrics' ) self . redis_conn . rename ( 'ionosphere.new_unique_metrics' , 'ionosphere.unique_metrics' ) manage_ionosphere_unique_metrics = False ionosphere_unique_metrics = [ ] except Exception as e : if str ( e ) == 'no such key' : logger . info ( 'could not rename Redis set ionosphere.new_unique_metrics to ionosphere.unique_metrics: %s' % str ( e ) ) else : logger . error ( 'error :: could not rename Redis set ionosphere.new_unique_metrics to ionosphere.unique_metrics: %s' % str ( e ) ) try : self . redis_conn . setex ( 'ionosphere.manage_ionosphere_unique_metrics' , 300 , time ( ) ) except : logger . error ( 'error :: failed to set key :: ionosphere.manage_ionosphere_unique_metrics' ) redis_ionosphere_unique_metrics = [ ] try : redis_ionosphere_unique_metrics = self . redis_conn_decoded . smembers ( 'ionosphere.unique_metrics' ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: failed to get ionosphere.unique_metrics from Redis' ) if redis_ionosphere_unique_metrics is not None : ionosphere_unique_metrics = list ( redis_ionosphere_unique_metrics ) ionosphere_unique_metrics_count = len ( ionosphere_unique_metrics ) logger . info ( 'the new Redis ionosphere.unique_metrics set has %s metrics' % ( str ( ionosphere_unique_metrics_count ) ) ) else : logger . info ( 'Redis ionosphere.unique_metrics unknown setting to []' ) ionosphere_unique_metrics = [ ] if engine : try : engine . dispose ( ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: calling engine.dispose()' ) ionosphere_untrainable_metrics = [ ] ionosphere_untrainable_metrics_redis_set = 'ionosphere.untrainable_metrics' try : ionosphere_untrainable_metrics = list ( self . redis_conn_decoded . smembers ( ionosphere_untrainable_metrics_redis_set ) ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: could not get the ionosphere.untrainable_metrics set from Redis' ) if ionosphere_untrainable_metrics : ionosphere_untrainable_metrics_check_time = int ( time ( ) ) for ionosphere_untrainable_metric_str in ionosphere_untrainable_metrics : try : ionosphere_untrainable_metric = literal_eval ( ionosphere_untrainable_metric_str ) ium_remove_after_timestamp = int ( ionosphere_untrainable_metric [ 6 ] ) if ionosphere_untrainable_metrics_check_time >= ium_remove_after_timestamp : try : self . redis_conn . srem ( ionosphere_untrainable_metrics_redis_set , str ( ionosphere_untrainable_metric ) ) logger . info ( 'removed item - %s - from Redis set - %s' % ( str ( ionosphere_untrainable_metric ) , ionosphere_untrainable_metrics_redis_set ) ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: failed to remove item list from Redis set - %s' % ionosphere_untrainable_metrics_redis_set ) except : logger . error ( traceback . format_exc ( ) ) logger . error ( 'error :: failed to manage Redis set %s' % ionosphere_untrainable_metrics_redis_set ) return True
short_docstring: Create a Redis set of all Ionosphere enabled metrics.
short_docstring_tokens: Create a Redis set of all Ionosphere enabled metrics .
comment: """
          - Create a Redis set of all Ionosphere enabled metrics.
          - Manage the ionosphere.untrainable_metrics set, removing items when
            they 'expire'
  
          :param i: python process id
          :return: returns True
  
          """$SEP$# ionosphere_unique_metrics = list(self.redis_conn.smembers('ionosphere.unique_metrics'))$SEP$# @modified 20191030 - Bug #3266: py3 Redis binary objects not strings$SEP$#                      Branch #3262: py3$SEP$# redis_ionosphere_unique_metrics = self.redis_conn.smembers('ionosphere.unique_metrics')$SEP$# ionosphere_unique_metrics = []$SEP$# Determine the metrics that have ionosphere_enabled$SEP$# @added 20170103 - Task #1658: Patterning Skyline Ionosphere$SEP$# TODO: We need 2 sets not just ionosphere.unique_metrics otherwise$SEP$#       if a metric is switch from Analyzer to Mirage will send all$SEP$#       matched anomalies to Ionosphere even if there is no features$SEP$#       profile at the specified duration.$SEP$# ionosphere.analyzer.unique_metrics (at FULL_DURATION)$SEP$# ionosphere.mirage.unique_metrics (NOT at FULL_DURATION)$SEP$# @modified 20170108 - Feature #1852: Ionosphere - features_profile matched graphite graphs$SEP$# Yes those ^^ are needed, MySQL join?$SEP$# @added 20190528 - Branch #3002: docker$SEP$# Testing the query was fine and Ionosphere metrics can go to 0 if$SEP$# all were disabled$SEP$# logger.info('added %s to ionosphere.new_unique_metrics Redis set' % metric_name)$SEP$# ionosphere_unique_metrics = list(self.redis_conn.smembers('ionosphere.unique_metrics'))$SEP$# @modified 20191030 - Bug #3266: py3 Redis binary objects not strings$SEP$#                      Branch #3262: py3$SEP$# redis_ionosphere_unique_metrics = self.redis_conn.smembers('ionosphere.unique_metrics')$SEP$# ionosphere_unique_metrics = []$SEP$# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics$SEP$# Manage ionosphere_untrainable_metrics Redis set