def _recomputeRecordFromKNN(self, record):
    """
    returns the classified labeling of record
    """
    inputs = {
      "categoryIn": [None],
      "bottomUpIn": self._getStateAnomalyVector(record),
    }

    outputs = {"categoriesOut": numpy.zeros((1,)),
               "bestPrototypeIndices":numpy.zeros((1,)),
               "categoryProbabilitiesOut":numpy.zeros((1,))}

    # Only use points before record to classify and after the wait period.
    classifier_indexes = numpy.array(
        self._knnclassifier.getParameter('categoryRecencyList'))
    valid_idx = numpy.where(
        (classifier_indexes >= self.getParameter('trainRecords')) &
        (classifier_indexes < record.ROWID)
      )[0].tolist()

    if len(valid_idx) == 0:
      return None

    self._knnclassifier.setParameter('inferenceMode', None, True)
    self._knnclassifier.setParameter('learningMode', None, False)
    self._knnclassifier.compute(inputs, outputs)
    self._knnclassifier.setParameter('learningMode', None, True)

    classifier_distances = self._knnclassifier.getLatestDistances()
    valid_distances = classifier_distances[valid_idx]
    if valid_distances.min() <= self._classificationMaxDist:
      classifier_indexes_prev = classifier_indexes[valid_idx]
      rowID = classifier_indexes_prev[valid_distances.argmin()]
      indexID = numpy.where(classifier_indexes == rowID)[0][0]
      category = self._knnclassifier.getCategoryList()[indexID]
      return category
    return None


def _labelToCategoryNumber(self, label):
    """
    Since the KNN Classifier stores categories as numbers, we must store each
    label as a number. This method converts from a label to a unique number.
    Each label is assigned a unique bit so multiple labels may be assigned to
    a single record.
    """
    if label not in self.saved_categories:
      self.saved_categories.append(label)
    return pow(2, self.saved_categories.index(label))

def _labelToCategoryNumber ( self , label ) : if label not in self . saved_categories : self . saved_categories . append ( label ) return pow ( 2 , self . saved_categories . index ( label ) )

Since the KNN Classifier stores categories as numbers we must store each label as a number . This method converts from a label to a unique number . Each label is assigned a unique bit so multiple labels may be assigned to a single record .


def _labelListToCategoryNumber(self, labelList):
    """
    This method takes a list of labels and returns a unique category number.
    This enables this class to store a list of categories for each point since
    the KNN classifier only stores a single number category for each record.
    """
    categoryNumber = 0
    for label in labelList:
      categoryNumber += self._labelToCategoryNumber(label)
    return categoryNumber

def _labelListToCategoryNumber ( self , labelList ) : categoryNumber = 0 for label in labelList : categoryNumber += self . _labelToCategoryNumber ( label ) return categoryNumber

This method takes a list of labels and returns a unique category number . This enables this class to store a list of categories for each point since the KNN classifier only stores a single number category for each record .


def _categoryToLabelList(self, category):
    """
    Converts a category number into a list of labels
    """
    if category is None:
      return []

    labelList = []
    labelNum = 0
    while category > 0:
      if category % 2 == 1:
        labelList.append(self.saved_categories[labelNum])
      labelNum += 1
      category = category >> 1
    return labelList

def _categoryToLabelList ( self , category ) : if category is None : return [ ] labelList = [ ] labelNum = 0 while category > 0 : if category % 2 == 1 : labelList . append ( self . saved_categories [ labelNum ] ) labelNum += 1 category = category >> 1 return labelList


def _getStateAnomalyVector(self, state):
    """
    Returns a state's anomaly vertor converting it from spare to dense
    """
    vector = numpy.zeros(self._anomalyVectorLength)
    vector[state.anomalyVector] = 1
    return vector

def _getStateAnomalyVector ( self , state ) : vector = numpy . zeros ( self . _anomalyVectorLength ) vector [ state . anomalyVector ] = 1 return vector

Returns a state s anomaly vertor converting it from spare to dense


def getLabels(self, start=None, end=None):
    """
    Get the labels on classified points within range start to end. Not inclusive
    of end.

    :returns: (dict) with format:

      ::

        {
          'isProcessing': boolean,
          'recordLabels': list of results
        }

      ``isProcessing`` - currently always false as recalculation blocks; used if
      reprocessing of records is still being performed;

      Each item in ``recordLabels`` is of format:

      ::

        {
          'ROWID': id of the row,
          'labels': list of strings
        }

    """
    if len(self._recordsCache) == 0:
      return {
        'isProcessing': False,
        'recordLabels': []
      }
    try:
      start = int(start)
    except Exception:
      start = 0

    try:
      end = int(end)
    except Exception:
      end = self._recordsCache[-1].ROWID

    if end <= start:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for 'getLabels'.",
                                                debugInfo={
          'requestRange': {
            'startRecordID': start,
            'endRecordID': end
          },
          'numRecordsStored': len(self._recordsCache)
        })

    results = {
      'isProcessing': False,
      'recordLabels': []
    }

    ROWIDX = numpy.array(
        self._knnclassifier.getParameter('categoryRecencyList'))
    validIdx = numpy.where((ROWIDX >= start) & (ROWIDX < end))[0].tolist()
    categories = self._knnclassifier.getCategoryList()
    for idx in validIdx:
      row = dict(
        ROWID=int(ROWIDX[idx]),
        labels=self._categoryToLabelList(categories[idx]))
      results['recordLabels'].append(row)

    return results

Get the labels on classified points within range start to end . Not inclusive of end .


def removeLabels(self, start=None, end=None, labelFilter=None):
    """
    Remove labels from each record with record ROWID in range from
    ``start`` to ``end``, noninclusive of end. Removes all records if
    ``labelFilter`` is None, otherwise only removes the labels equal to
    ``labelFilter``.

    This will recalculate all points from end to the last record stored in the
    internal cache of this classifier.

    :param start: (int) start index
    :param end: (int) end index (noninclusive)
    :param labelFilter: (string) label filter
    """
    if len(self._recordsCache) == 0:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'removeLabels'. Model has no saved records.")

    try:
      start = int(start)
    except Exception:
      start = 0

    try:
      end = int(end)
    except Exception:
      end = self._recordsCache[-1].ROWID

    startID = self._recordsCache[0].ROWID

    clippedStart = 0 if start is None else max(0, start - startID)
    clippedEnd = len(self._recordsCache) if end is None else \
      max(0, min( len( self._recordsCache) , end - startID))

    if clippedEnd <= clippedStart:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'removeLabels'.", debugInfo={
          'requestRange': {
            'startRecordID': start,
            'endRecordID': end
          },
          'clippedRequestRange': {
            'startRecordID': clippedStart,
            'endRecordID': clippedEnd
          },
          'validRange': {
            'startRecordID': startID,
            'endRecordID': self._recordsCache[len(self._recordsCache)-1].ROWID
          },
          'numRecordsStored': len(self._recordsCache)
        })

    # Remove records within the cache
    recordsToDelete = []
    for state in self._recordsCache[clippedStart:clippedEnd]:
      if labelFilter is not None:
        if labelFilter in state.anomalyLabel:
          state.anomalyLabel.remove(labelFilter)
      else:
        state.anomalyLabel = []
      state.setByUser = False
      recordsToDelete.append(state)
    self._deleteRecordsFromKNN(recordsToDelete)

    # Remove records not in cache
    self._deleteRangeFromKNN(start, end)

    # Recompute [clippedEnd, ...)
    for state in self._recordsCache[clippedEnd:]:
      self._classifyState(state)

Remove labels from each record with record ROWID in range from start to end noninclusive of end . Removes all records if labelFilter is None otherwise only removes the labels equal to labelFilter .


def match(self, record):
    '''
    Returns True if the record matches any of the provided filters
    '''

    for field, meta in self.filterDict.iteritems():
      index = meta['index']
      categories = meta['categories']
      for category in categories:
        # Record might be blank, handle this
        if not record:
          continue
        if record[index].find(category) != -1:
          '''
          This field contains the string we're searching for
          so we'll keep the records
          '''
          return True

    # None of the categories were found in this record
    return False


def stripUnlearnedColumns(self, activeArray):
    """
    Removes the set of columns who have never been active from the set of
    active columns selected in the inhibition round. Such columns cannot
    represent learned pattern and are therefore meaningless if only inference
    is required. This should not be done when using a random, unlearned SP
    since you would end up with no active columns.

    :param activeArray: An array whose size is equal to the number of columns.
        Any columns marked as active with an activeDutyCycle of 0 have
        never been activated before and therefore are not active due to
        learning. Any of these (unlearned) columns will be disabled (set to 0).
    """
    neverLearned = numpy.where(self._activeDutyCycles == 0)[0]
    activeArray[neverLearned] = 0

def stripUnlearnedColumns ( self , activeArray ) : neverLearned = numpy . where ( self . _activeDutyCycles == 0 ) [ 0 ] activeArray [ neverLearned ] = 0

Removes the set of columns who have never been active from the set of active columns selected in the inhibition round . Such columns cannot represent learned pattern and are therefore meaningless if only inference is required . This should not be done when using a random unlearned SP since you would end up with no active columns .


def _updateMinDutyCycles(self):
    """
    Updates the minimum duty cycles defining normal activity for a column. A
    column with activity duty cycle below this minimum threshold is boosted.
    """
    if self._globalInhibition or self._inhibitionRadius > self._numInputs:
      self._updateMinDutyCyclesGlobal()
    else:
      self._updateMinDutyCyclesLocal()

def _updateMinDutyCycles ( self ) : if self . _globalInhibition or self . _inhibitionRadius > self . _numInputs : self . _updateMinDutyCyclesGlobal ( ) else : self . _updateMinDutyCyclesLocal ( )

Updates the minimum duty cycles defining normal activity for a column . A column with activity duty cycle below this minimum threshold is boosted .


def _updateMinDutyCyclesGlobal(self):
    """
    Updates the minimum duty cycles in a global fashion. Sets the minimum duty
    cycles for the overlap all columns to be a percent of the maximum in the
    region, specified by minPctOverlapDutyCycle. Functionality it is equivalent
    to _updateMinDutyCyclesLocal, but this function exploits the globality of
    the computation to perform it in a straightforward, and efficient manner.
    """
    self._minOverlapDutyCycles.fill(
        self._minPctOverlapDutyCycles * self._overlapDutyCycles.max()
      )

def _updateMinDutyCyclesGlobal ( self ) : self . _minOverlapDutyCycles . fill ( self . _minPctOverlapDutyCycles * self . _overlapDutyCycles . max ( ) )

Updates the minimum duty cycles in a global fashion . Sets the minimum duty cycles for the overlap all columns to be a percent of the maximum in the region specified by minPctOverlapDutyCycle . Functionality it is equivalent to _updateMinDutyCyclesLocal but this function exploits the globality of the computation to perform it in a straightforward and efficient manner .


def _updateMinDutyCyclesLocal(self):
    """
    Updates the minimum duty cycles. The minimum duty cycles are determined
    locally. Each column's minimum duty cycles are set to be a percent of the
    maximum duty cycles in the column's neighborhood. Unlike
    _updateMinDutyCyclesGlobal, here the values can be quite different for
    different columns.
    """
    for column in xrange(self._numColumns):
      neighborhood = self._getColumnNeighborhood(column)

      maxActiveDuty = self._activeDutyCycles[neighborhood].max()
      maxOverlapDuty = self._overlapDutyCycles[neighborhood].max()

      self._minOverlapDutyCycles[column] = (maxOverlapDuty *
                                            self._minPctOverlapDutyCycles)

def _updateMinDutyCyclesLocal ( self ) : for column in xrange ( self . _numColumns ) : neighborhood = self . _getColumnNeighborhood ( column ) maxActiveDuty = self . _activeDutyCycles [ neighborhood ] . max ( ) maxOverlapDuty = self . _overlapDutyCycles [ neighborhood ] . max ( ) self . _minOverlapDutyCycles [ column ] = ( maxOverlapDuty * self . _minPctOverlapDutyCycles )

Updates the minimum duty cycles . The minimum duty cycles are determined locally . Each column s minimum duty cycles are set to be a percent of the maximum duty cycles in the column s neighborhood . Unlike _updateMinDutyCyclesGlobal here the values can be quite different for different columns .


def _updateDutyCycles(self, overlaps, activeColumns):
    """
    Updates the duty cycles for each column. The OVERLAP duty cycle is a moving
    average of the number of inputs which overlapped with the each column. The
    ACTIVITY duty cycles is a moving average of the frequency of activation for
    each column.

    Parameters:
    ----------------------------
    :param overlaps:
                    An array containing the overlap score for each column.
                    The overlap score for a column is defined as the number
                    of synapses in a "connected state" (connected synapses)
                    that are connected to input bits which are turned on.
    :param activeColumns:
                    An array containing the indices of the active columns,
                    the sparse set of columns which survived inhibition
    """
    overlapArray = numpy.zeros(self._numColumns, dtype=realDType)
    activeArray = numpy.zeros(self._numColumns, dtype=realDType)
    overlapArray[overlaps > 0] = 1
    activeArray[activeColumns] = 1

    period = self._dutyCyclePeriod
    if (period > self._iterationNum):
      period = self._iterationNum

    self._overlapDutyCycles = self._updateDutyCyclesHelper(
                                self._overlapDutyCycles,
                                overlapArray,
                                period
                              )

    self._activeDutyCycles = self._updateDutyCyclesHelper(
                                self._activeDutyCycles,
                                activeArray,
                                period
                              )

def _updateDutyCycles ( self , overlaps , activeColumns ) : overlapArray = numpy . zeros ( self . _numColumns , dtype = realDType ) activeArray = numpy . zeros ( self . _numColumns , dtype = realDType ) overlapArray [ overlaps > 0 ] = 1 activeArray [ activeColumns ] = 1 period = self . _dutyCyclePeriod if ( period > self . _iterationNum ) : period = self . _iterationNum self . _overlapDutyCycles = self . _updateDutyCyclesHelper ( self . _overlapDutyCycles , overlapArray , period ) self . _activeDutyCycles = self . _updateDutyCyclesHelper ( self . _activeDutyCycles , activeArray , period )

Updates the duty cycles for each column . The OVERLAP duty cycle is a moving average of the number of inputs which overlapped with the each column . The ACTIVITY duty cycles is a moving average of the frequency of activation for each column .


def _avgColumnsPerInput(self):
    """
    The average number of columns per input, taking into account the topology
    of the inputs and columns. This value is used to calculate the inhibition
    radius. This function supports an arbitrary number of dimensions. If the
    number of column dimensions does not match the number of input dimensions,
    we treat the missing, or phantom dimensions as 'ones'.
    """
    #TODO: extend to support different number of dimensions for inputs and
    # columns
    numDim = max(self._columnDimensions.size, self._inputDimensions.size)
    colDim = numpy.ones(numDim)
    colDim[:self._columnDimensions.size] = self._columnDimensions

    inputDim = numpy.ones(numDim)
    inputDim[:self._inputDimensions.size] = self._inputDimensions

    columnsPerInput = colDim.astype(realDType) / inputDim
    return numpy.average(columnsPerInput)

The average number of columns per input taking into account the topology of the inputs and columns . This value is used to calculate the inhibition radius . This function supports an arbitrary number of dimensions . If the number of column dimensions does not match the number of input dimensions we treat the missing or phantom dimensions as ones .


def _avgConnectedSpanForColumn1D(self, columnIndex):
    """
    The range of connected synapses for column. This is used to
    calculate the inhibition radius. This variation of the function only
    supports a 1 dimensional column topology.

    Parameters:
    ----------------------------
    :param columnIndex:   The index identifying a column in the permanence,
                          potential and connectivity matrices
    """
    assert(self._inputDimensions.size == 1)
    connected = self._connectedSynapses[columnIndex].nonzero()[0]
    if connected.size == 0:
      return 0
    else:
      return max(connected) - min(connected) + 1

def _avgConnectedSpanForColumn1D ( self , columnIndex ) : assert ( self . _inputDimensions . size == 1 ) connected = self . _connectedSynapses [ columnIndex ] . nonzero ( ) [ 0 ] if connected . size == 0 : return 0 else : return max ( connected ) - min ( connected ) + 1

The range of connected synapses for column . This is used to calculate the inhibition radius . This variation of the function only supports a 1 dimensional column topology .


def _avgConnectedSpanForColumn2D(self, columnIndex):
    """
    The range of connectedSynapses per column, averaged for each dimension.
    This value is used to calculate the inhibition radius. This variation of
    the  function only supports a 2 dimensional column topology.

    Parameters:
    ----------------------------
    :param columnIndex:   The index identifying a column in the permanence,
                          potential and connectivity matrices
    """
    assert(self._inputDimensions.size == 2)
    connected = self._connectedSynapses[columnIndex]
    (rows, cols) = connected.reshape(self._inputDimensions).nonzero()
    if  rows.size == 0 and cols.size == 0:
      return 0
    rowSpan = rows.max() - rows.min() + 1
    colSpan = cols.max() - cols.min() + 1
    return numpy.average([rowSpan, colSpan])

def _avgConnectedSpanForColumn2D ( self , columnIndex ) : assert ( self . _inputDimensions . size == 2 ) connected = self . _connectedSynapses [ columnIndex ] ( rows , cols ) = connected . reshape ( self . _inputDimensions ) . nonzero ( ) if rows . size == 0 and cols . size == 0 : return 0 rowSpan = rows . max ( ) - rows . min ( ) + 1 colSpan = cols . max ( ) - cols . min ( ) + 1 return numpy . average ( [ rowSpan , colSpan ] )

The range of connectedSynapses per column averaged for each dimension . This value is used to calculate the inhibition radius . This variation of the function only supports a 2 dimensional column topology .


def _bumpUpWeakColumns(self):
    """
    This method increases the permanence values of synapses of columns whose
    activity level has been too low. Such columns are identified by having an
    overlap duty cycle that drops too much below those of their peers. The
    permanence values for such columns are increased.
    """
    weakColumns = numpy.where(self._overlapDutyCycles
                                < self._minOverlapDutyCycles)[0]
    for columnIndex in weakColumns:
      perm = self._permanences[columnIndex].astype(realDType)
      maskPotential = numpy.where(self._potentialPools[columnIndex] > 0)[0]
      perm[maskPotential] += self._synPermBelowStimulusInc
      self._updatePermanencesForColumn(perm, columnIndex, raisePerm=False)

def _bumpUpWeakColumns ( self ) : weakColumns = numpy . where ( self . _overlapDutyCycles < self . _minOverlapDutyCycles ) [ 0 ] for columnIndex in weakColumns : perm = self . _permanences [ columnIndex ] . astype ( realDType ) maskPotential = numpy . where ( self . _potentialPools [ columnIndex ] > 0 ) [ 0 ] perm [ maskPotential ] += self . _synPermBelowStimulusInc self . _updatePermanencesForColumn ( perm , columnIndex , raisePerm = False )

This method increases the permanence values of synapses of columns whose activity level has been too low . Such columns are identified by having an overlap duty cycle that drops too much below those of their peers . The permanence values for such columns are increased .


def _raisePermanenceToThreshold(self, perm, mask):
    """
    This method ensures that each column has enough connections to input bits
    to allow it to become active. Since a column must have at least
    'self._stimulusThreshold' overlaps in order to be considered during the
    inhibition phase, columns without such minimal number of connections, even
    if all the input bits they are connected to turn on, have no chance of
    obtaining the minimum threshold. For such columns, the permanence values
    are increased until the minimum number of connections are formed.

    Parameters:
    ----------------------------
    :param perm:    An array of permanence values for a column. The array is
                    "dense", i.e. it contains an entry for each input bit, even
                    if the permanence value is 0.
    :param mask:    the indices of the columns whose permanences need to be
                    raised.
    """
    if len(mask) < self._stimulusThreshold:
      raise Exception("This is likely due to a " +
      "value of stimulusThreshold that is too large relative " +
      "to the input size. [len(mask) < self._stimulusThreshold]")

    numpy.clip(perm, self._synPermMin, self._synPermMax, out=perm)
    while True:
      numConnected = numpy.nonzero(
        perm > self._synPermConnected - PERMANENCE_EPSILON)[0].size

      if numConnected >= self._stimulusThreshold:
        return
      perm[mask] += self._synPermBelowStimulusInc

This method ensures that each column has enough connections to input bits to allow it to become active . Since a column must have at least self . _stimulusThreshold overlaps in order to be considered during the inhibition phase columns without such minimal number of connections even if all the input bits they are connected to turn on have no chance of obtaining the minimum threshold . For such columns the permanence values are increased until the minimum number of connections are formed .


def _initPermConnected(self):
    """
    Returns a randomly generated permanence value for a synapses that is
    initialized in a connected state. The basic idea here is to initialize
    permanence values very close to synPermConnected so that a small number of
    learning steps could make it disconnected or connected.

    Note: experimentation was done a long time ago on the best way to initialize
    permanence values, but the history for this particular scheme has been lost.
    """
    p = self._synPermConnected + (
        self._synPermMax - self._synPermConnected)*self._random.getReal64()

    # Ensure we don't have too much unnecessary precision. A full 64 bits of
    # precision causes numerical stability issues across platforms and across
    # implementations
    p = int(p*100000) / 100000.0
    return p

Returns a randomly generated permanence value for a synapses that is initialized in a connected state . The basic idea here is to initialize permanence values very close to synPermConnected so that a small number of learning steps could make it disconnected or connected .


def _initPermNonConnected(self):
    """
    Returns a randomly generated permanence value for a synapses that is to be
    initialized in a non-connected state.
    """
    p = self._synPermConnected * self._random.getReal64()

    # Ensure we don't have too much unnecessary precision. A full 64 bits of
    # precision causes numerical stability issues across platforms and across
    # implementations
    p = int(p*100000) / 100000.0
    return p

Returns a randomly generated permanence value for a synapses that is to be initialized in a non - connected state .


def _initPermanence(self, potential, connectedPct):
    """
    Initializes the permanences of a column. The method
    returns a 1-D array the size of the input, where each entry in the
    array represents the initial permanence value between the input bit
    at the particular index in the array, and the column represented by
    the 'index' parameter.

    Parameters:
    ----------------------------
    :param potential: A numpy array specifying the potential pool of the column.
                    Permanence values will only be generated for input bits
                    corresponding to indices for which the mask value is 1.
    :param connectedPct: A value between 0 or 1 governing the chance, for each
                         permanence, that the initial permanence value will
                         be a value that is considered connected.
    """
    # Determine which inputs bits will start out as connected
    # to the inputs. Initially a subset of the input bits in a
    # column's potential pool will be connected. This number is
    # given by the parameter "connectedPct"
    perm = numpy.zeros(self._numInputs, dtype=realDType)
    for i in xrange(self._numInputs):
      if (potential[i] < 1):
        continue

      if (self._random.getReal64() <= connectedPct):
        perm[i] = self._initPermConnected()
      else:
        perm[i] = self._initPermNonConnected()

    # Clip off low values. Since we use a sparse representation
    # to store the permanence values this helps reduce memory
    # requirements.
    perm[perm < self._synPermTrimThreshold] = 0

    return perm

Initializes the permanences of a column . The method returns a 1 - D array the size of the input where each entry in the array represents the initial permanence value between the input bit at the particular index in the array and the column represented by the index parameter .


def _updateBoostFactorsGlobal(self):
    """
    Update boost factors when global inhibition is used
    """
    # When global inhibition is enabled, the target activation level is
    # the sparsity of the spatial pooler
    if (self._localAreaDensity > 0):
      targetDensity = self._localAreaDensity
    else:
      inhibitionArea = ((2 * self._inhibitionRadius + 1)
                        ** self._columnDimensions.size)
      inhibitionArea = min(self._numColumns, inhibitionArea)
      targetDensity = float(self._numActiveColumnsPerInhArea) / inhibitionArea
      targetDensity = min(targetDensity, 0.5)

    self._boostFactors = numpy.exp(
      (targetDensity - self._activeDutyCycles) * self._boostStrength)


def _updateBoostFactorsLocal(self):
    """
    Update boost factors when local inhibition is used
    """
    # Determine the target activation level for each column
    # The targetDensity is the average activeDutyCycles of the neighboring
    # columns of each column.
    targetDensity = numpy.zeros(self._numColumns, dtype=realDType)
    for i in xrange(self._numColumns):
      maskNeighbors = self._getColumnNeighborhood(i)
      targetDensity[i] = numpy.mean(self._activeDutyCycles[maskNeighbors])

    self._boostFactors = numpy.exp(
      (targetDensity - self._activeDutyCycles) * self._boostStrength)


def _inhibitColumns(self, overlaps):
    """
    Performs inhibition. This method calculates the necessary values needed to
    actually perform inhibition and then delegates the task of picking the
    active columns to helper functions.

    Parameters:
    ----------------------------
    :param overlaps: an array containing the overlap score for each  column.
                    The overlap score for a column is defined as the number
                    of synapses in a "connected state" (connected synapses)
                    that are connected to input bits which are turned on.
    """
    # determine how many columns should be selected in the inhibition phase.
    # This can be specified by either setting the 'numActiveColumnsPerInhArea'
    # parameter or the 'localAreaDensity' parameter when initializing the class
    if (self._localAreaDensity > 0):
      density = self._localAreaDensity
    else:
      inhibitionArea = ((2*self._inhibitionRadius + 1)
                                    ** self._columnDimensions.size)
      inhibitionArea = min(self._numColumns, inhibitionArea)
      density = float(self._numActiveColumnsPerInhArea) / inhibitionArea
      density = min(density, 0.5)

    if self._globalInhibition or \
      self._inhibitionRadius > max(self._columnDimensions):
      return self._inhibitColumnsGlobal(overlaps, density)
    else:
      return self._inhibitColumnsLocal(overlaps, density)

Performs inhibition . This method calculates the necessary values needed to actually perform inhibition and then delegates the task of picking the active columns to helper functions .


def _inhibitColumnsGlobal(self, overlaps, density):
    """
    Perform global inhibition. Performing global inhibition entails picking the
    top 'numActive' columns with the highest overlap score in the entire
    region. At most half of the columns in a local neighborhood are allowed to
    be active. Columns with an overlap score below the 'stimulusThreshold' are
    always inhibited.

    :param overlaps: an array containing the overlap score for each  column.
                    The overlap score for a column is defined as the number
                    of synapses in a "connected state" (connected synapses)
                    that are connected to input bits which are turned on.
    :param density: The fraction of columns to survive inhibition.
    @return list with indices of the winning columns
    """
    #calculate num active per inhibition area
    numActive = int(density * self._numColumns)

    # Calculate winners using stable sort algorithm (mergesort)
    # for compatibility with C++
    sortedWinnerIndices = numpy.argsort(overlaps, kind='mergesort')

    # Enforce the stimulus threshold
    start = len(sortedWinnerIndices) - numActive
    while start < len(sortedWinnerIndices):
      i = sortedWinnerIndices[start]
      if overlaps[i] >= self._stimulusThreshold:
        break
      else:
        start += 1

    return sortedWinnerIndices[start:][::-1]

Perform global inhibition . Performing global inhibition entails picking the top numActive columns with the highest overlap score in the entire region . At most half of the columns in a local neighborhood are allowed to be active . Columns with an overlap score below the stimulusThreshold are always inhibited .


def _inhibitColumnsLocal(self, overlaps, density):
    """
    Performs local inhibition. Local inhibition is performed on a column by
    column basis. Each column observes the overlaps of its neighbors and is
    selected if its overlap score is within the top 'numActive' in its local
    neighborhood. At most half of the columns in a local neighborhood are
    allowed to be active. Columns with an overlap score below the
    'stimulusThreshold' are always inhibited.

    :param overlaps: an array containing the overlap score for each  column.
                    The overlap score for a column is defined as the number
                    of synapses in a "connected state" (connected synapses)
                    that are connected to input bits which are turned on.
    :param density: The fraction of columns to survive inhibition. This
                    value is only an intended target. Since the surviving
                    columns are picked in a local fashion, the exact fraction
                    of surviving columns is likely to vary.
    @return list with indices of the winning columns
    """

    activeArray = numpy.zeros(self._numColumns, dtype="bool")

    for column, overlap in enumerate(overlaps):
      if overlap >= self._stimulusThreshold:
        neighborhood = self._getColumnNeighborhood(column)
        neighborhoodOverlaps = overlaps[neighborhood]

        numBigger = numpy.count_nonzero(neighborhoodOverlaps > overlap)

        # When there is a tie, favor neighbors that are already selected as
        # active.
        ties = numpy.where(neighborhoodOverlaps == overlap)
        tiedNeighbors = neighborhood[ties]
        numTiesLost = numpy.count_nonzero(activeArray[tiedNeighbors])

        numActive = int(0.5 + density * len(neighborhood))
        if numBigger + numTiesLost < numActive:
          activeArray[column] = True

    return activeArray.nonzero()[0]

Performs local inhibition . Local inhibition is performed on a column by column basis . Each column observes the overlaps of its neighbors and is selected if its overlap score is within the top numActive in its local neighborhood . At most half of the columns in a local neighborhood are allowed to be active . Columns with an overlap score below the stimulusThreshold are always inhibited .


def _getColumnNeighborhood(self, centerColumn):
    """
    Gets a neighborhood of columns.

    Simply calls topology.neighborhood or topology.wrappingNeighborhood

    A subclass can insert different topology behavior by overriding this method.

    :param centerColumn (int)
    The center of the neighborhood.

    @returns (1D numpy array of integers)
    The columns in the neighborhood.
    """
    if self._wrapAround:
      return topology.wrappingNeighborhood(centerColumn,
                                           self._inhibitionRadius,
                                           self._columnDimensions)

    else:
      return topology.neighborhood(centerColumn,
                                   self._inhibitionRadius,
                                   self._columnDimensions)

def _getColumnNeighborhood ( self , centerColumn ) : if self . _wrapAround : return topology . wrappingNeighborhood ( centerColumn , self . _inhibitionRadius , self . _columnDimensions ) else : return topology . neighborhood ( centerColumn , self . _inhibitionRadius , self . _columnDimensions )


def _getInputNeighborhood(self, centerInput):
    """
    Gets a neighborhood of inputs.

    Simply calls topology.wrappingNeighborhood or topology.neighborhood.

    A subclass can insert different topology behavior by overriding this method.

    :param centerInput (int)
    The center of the neighborhood.

    @returns (1D numpy array of integers)
    The inputs in the neighborhood.
    """
    if self._wrapAround:
      return topology.wrappingNeighborhood(centerInput,
                                           self._potentialRadius,
                                           self._inputDimensions)
    else:
      return topology.neighborhood(centerInput,
                                   self._potentialRadius,
                                   self._inputDimensions)

def _getInputNeighborhood ( self , centerInput ) : if self . _wrapAround : return topology . wrappingNeighborhood ( centerInput , self . _potentialRadius , self . _inputDimensions ) else : return topology . neighborhood ( centerInput , self . _potentialRadius , self . _inputDimensions )


def Array(dtype, size=None, ref=False):
  """Factory function that creates typed Array or ArrayRef objects

  dtype - the data type of the array (as string).
    Supported types are: Byte, Int16, UInt16, Int32, UInt32, Int64, UInt64, Real32, Real64

  size - the size of the array. Must be positive integer.
  """

  def getArrayType(self):
    """A little function to replace the getType() method of arrays

    It returns a string representation of the array element type instead of the
    integer value (NTA_BasicType enum) returned by the origianl array
    """
    return self._dtype

  # ArrayRef can't be allocated
  if ref:
    assert size is None

  index = basicTypes.index(dtype)
  if index == -1:
    raise Exception('Invalid data type: ' + dtype)
  if size and size <= 0:
    raise Exception('Array size must be positive')
  suffix = 'ArrayRef' if ref else 'Array'
  arrayFactory = getattr(engine_internal, dtype + suffix)
  arrayFactory.getType = getArrayType

  if size:
    a = arrayFactory(size)
  else:
    a = arrayFactory()

  a._dtype = basicTypes[index]
  return a


def getInputNames(self):
    """
    Returns list of input names in spec.
    """
    inputs = self.getSpec().inputs
    return [inputs.getByIndex(i)[0] for i in xrange(inputs.getCount())]

def getInputNames ( self ) : inputs = self . getSpec ( ) . inputs return [ inputs . getByIndex ( i ) [ 0 ] for i in xrange ( inputs . getCount ( ) ) ]


def getOutputNames(self):
    """
    Returns list of output names in spec.
    """
    outputs = self.getSpec().outputs
    return [outputs.getByIndex(i)[0] for i in xrange(outputs.getCount())]

def getOutputNames ( self ) : outputs = self . getSpec ( ) . outputs return [ outputs . getByIndex ( i ) [ 0 ] for i in xrange ( outputs . getCount ( ) ) ]


def getParameter(self, paramName):
    """Get parameter value"""
    (setter, getter) = self._getParameterMethods(paramName)
    if getter is None:
      import exceptions
      raise exceptions.Exception(
          "getParameter -- parameter name '%s' does not exist in region %s of type %s"
          % (paramName, self.name, self.type))
    return getter(paramName)


def setParameter(self, paramName, value):
    """Set parameter value"""
    (setter, getter) = self._getParameterMethods(paramName)
    if setter is None:
      import exceptions
      raise exceptions.Exception(
          "setParameter -- parameter name '%s' does not exist in region %s of type %s"
          % (paramName, self.name, self.type))
    setter(paramName, value)


def _getRegions(self):
    """Get the collection of regions in a network

    This is a tricky one. The collection of regions returned from
    from the internal network is a collection of internal regions.
    The desired collection is a collelcion of net.Region objects
    that also points to this network (net.network) and not to
    the internal network. To achieve that a CollectionWrapper
    class is used with a custom makeRegion() function (see bellow)
    as a value wrapper. The CollectionWrapper class wraps each value in the
    original collection with the result of the valueWrapper.
    """

    def makeRegion(name, r):
      """Wrap a engine region with a nupic.engine_internal.Region

      Also passes the containing nupic.engine_internal.Network network in _network. This
      function is passed a value wrapper to the CollectionWrapper
      """
      r = Region(r, self)
      #r._network = self
      return r

    regions = CollectionWrapper(engine_internal.Network.getRegions(self), makeRegion)
    return regions


def writeToProto(self, proto):
    """
    Write state to proto object.

    :param proto: SDRClassifierRegionProto capnproto object
    """
    proto.implementation = self.implementation
    proto.steps = self.steps
    proto.alpha = self.alpha
    proto.verbosity = self.verbosity
    proto.maxCategoryCount = self.maxCategoryCount
    proto.learningMode = self.learningMode
    proto.inferenceMode = self.inferenceMode
    proto.recordNum = self.recordNum

    self._sdrClassifier.write(proto.sdrClassifier)

def writeToProto ( self , proto ) : proto . implementation = self . implementation proto . steps = self . steps proto . alpha = self . alpha proto . verbosity = self . verbosity proto . maxCategoryCount = self . maxCategoryCount proto . learningMode = self . learningMode proto . inferenceMode = self . inferenceMode proto . recordNum = self . recordNum self . _sdrClassifier . write ( proto . sdrClassifier )


def readFromProto(cls, proto):
    """
    Read state from proto object.

    :param proto: SDRClassifierRegionProto capnproto object
    """
    instance = cls()

    instance.implementation = proto.implementation
    instance.steps = proto.steps
    instance.stepsList = [int(i) for i in proto.steps.split(",")]
    instance.alpha = proto.alpha
    instance.verbosity = proto.verbosity
    instance.maxCategoryCount = proto.maxCategoryCount

    instance._sdrClassifier = SDRClassifierFactory.read(proto)

    instance.learningMode = proto.learningMode
    instance.inferenceMode = proto.inferenceMode
    instance.recordNum = proto.recordNum

    return instance

def readFromProto ( cls , proto ) : instance = cls ( ) instance . implementation = proto . implementation instance . steps = proto . steps instance . stepsList = [ int ( i ) for i in proto . steps . split ( "," ) ] instance . alpha = proto . alpha instance . verbosity = proto . verbosity instance . maxCategoryCount = proto . maxCategoryCount instance . _sdrClassifier = SDRClassifierFactory . read ( proto ) instance . learningMode = proto . learningMode instance . inferenceMode = proto . inferenceMode instance . recordNum = proto . recordNum return instance


def run(self):
    """ Runs the OPF Model

    Parameters:
    -------------------------------------------------------------------------
    retval:  (completionReason, completionMsg)
              where completionReason is one of the ClientJobsDAO.CMPL_REASON_XXX
                equates.
    """
    # -----------------------------------------------------------------------
    # Load the experiment's description.py module
    descriptionPyModule = helpers.loadExperimentDescriptionScriptFromDir(
      self._experimentDir)
    expIface = helpers.getExperimentDescriptionInterfaceFromModule(
      descriptionPyModule)
    expIface.normalizeStreamSources()

    modelDescription = expIface.getModelDescription()
    self._modelControl = expIface.getModelControl()

    # -----------------------------------------------------------------------
    # Create the input data stream for this task
    streamDef = self._modelControl['dataset']

    from nupic.data.stream_reader import StreamReader
    readTimeout = 0

    self._inputSource = StreamReader(streamDef, isBlocking=False,
                                     maxTimeout=readTimeout)

    # -----------------------------------------------------------------------
    #Get field statistics from the input source
    fieldStats = self._getFieldStats()
    # -----------------------------------------------------------------------
    # Construct the model instance
    self._model = ModelFactory.create(modelDescription)
    self._model.setFieldStatistics(fieldStats)
    self._model.enableLearning()
    self._model.enableInference(self._modelControl.get("inferenceArgs", None))

    # -----------------------------------------------------------------------
    # Instantiate the metrics
    self.__metricMgr = MetricsManager(self._modelControl.get('metrics',None),
                                      self._model.getFieldInfo(),
                                      self._model.getInferenceType())

    self.__loggedMetricPatterns = self._modelControl.get("loggedMetrics", [])

    self._optimizedMetricLabel = self.__getOptimizedMetricLabel()
    self._reportMetricLabels = matchPatterns(self._reportKeyPatterns,
                                              self._getMetricLabels())

    # -----------------------------------------------------------------------
    # Initialize periodic activities (e.g., for model result updates)
    self._periodic = self._initPeriodicActivities()

    # -----------------------------------------------------------------------
    # Create our top-level loop-control iterator
    numIters = self._modelControl.get('iterationCount', -1)

    # Are we asked to turn off learning for a certain # of iterations near the
    #  end?
    learningOffAt = None
    iterationCountInferOnly = self._modelControl.get('iterationCountInferOnly', 0)
    if iterationCountInferOnly == -1:
      self._model.disableLearning()
    elif iterationCountInferOnly > 0:
      assert numIters > iterationCountInferOnly, "when iterationCountInferOnly " \
        "is specified, iterationCount must be greater than " \
        "iterationCountInferOnly."
      learningOffAt = numIters - iterationCountInferOnly

    self.__runTaskMainLoop(numIters, learningOffAt=learningOffAt)

    # -----------------------------------------------------------------------
    # Perform final operations for model
    self._finalize()

    return (self._cmpReason, None)


def __runTaskMainLoop(self, numIters, learningOffAt=None):
    """ Main loop of the OPF Model Runner.

    Parameters:
    -----------------------------------------------------------------------

    recordIterator:    Iterator for counting number of records (see _runTask)
    learningOffAt:     If not None, learning is turned off when we reach this
                        iteration number

    """

    ## Reset sequence states in the model, so it starts looking for a new
    ## sequence
    self._model.resetSequenceStates()

    self._currentRecordIndex = -1
    while True:

      # If killed by a terminator, stop running
      if self._isKilled:
        break

      # If job stops or hypersearch ends, stop running
      if self._isCanceled:
        break

      # If the process is about to be killed, set as orphaned
      if self._isInterrupted.isSet():
        self.__setAsOrphaned()
        break

      # If model is mature, stop running ONLY IF  we are not the best model
      # for the job. Otherwise, keep running so we can keep returning
      # predictions to the user
      if self._isMature:
        if not self._isBestModel:
          self._cmpReason = self._jobsDAO.CMPL_REASON_STOPPED
          break
        else:
          self._cmpReason = self._jobsDAO.CMPL_REASON_EOF

      # Turn off learning?
        if learningOffAt is not None \
                  and self._currentRecordIndex == learningOffAt:
          self._model.disableLearning()

      # Read input record. Note that any failure here is a critical JOB failure
      #  and results in the job being immediately canceled and marked as
      #  failed. The runModelXXX code in hypesearch.utils, if it sees an
      #  exception of type utils.JobFailException, will cancel the job and
      #  copy the error message into the job record.
      try:
        inputRecord = self._inputSource.getNextRecordDict()
        if self._currentRecordIndex < 0:
          self._inputSource.setTimeout(10)
      except Exception, e:
        raise utils.JobFailException(ErrorCodes.streamReading, str(e.args),
                                     traceback.format_exc())

      if inputRecord is None:
        # EOF
        self._cmpReason = self._jobsDAO.CMPL_REASON_EOF
        break

      if inputRecord:
        # Process input record
        self._currentRecordIndex += 1

        result = self._model.run(inputRecord=inputRecord)

        # Compute metrics.
        result.metrics = self.__metricMgr.update(result)
        # If there are None, use defaults. see MetricsManager.getMetrics()
        # TODO remove this when JAVA API server is gone
        if not result.metrics:
          result.metrics = self.__metricMgr.getMetrics()

        # Write the result to the output cache. Don't write encodings, if they
        # were computed
        if InferenceElement.encodings in result.inferences:
          result.inferences.pop(InferenceElement.encodings)
        result.sensorInput.dataEncodings = None
        self._writePrediction(result)

        # Run periodic activities
        self._periodic.tick()

        if numIters >= 0 and self._currentRecordIndex >= numIters-1:
          break

      else:
        # Input source returned an empty record.
        #
        # NOTE: This is okay with Stream-based Source (when it times out
        # waiting for next record), but not okay with FileSource, which should
        # always return either with a valid record or None for EOF.
        raise ValueError("Got an empty record from FileSource: %r" %
                         inputRecord)


"Finished: modelID=%r; %r records processed. Performing final activities"

# Dump the experiment metrics at the end of the task

# Check if the current model is the best. Create a milestone if necessary

# If the model has been killed, it is not a candidate for "best model",

# and its output cache should be destroyed

Run final activities after a model has run. These include recording and
    logging the final score

Run final activities after a model has run . These include recording and logging the final score


def __createModelCheckpoint(self):
    """ Create a checkpoint from the current model, and store it in a dir named
    after checkpoint GUID, and finally store the GUID in the Models DB """

    if self._model is None or self._modelCheckpointGUID is None:
      return

    # Create an output store, if one doesn't exist already
    if self._predictionLogger is None:
      self._createPredictionLogger()

    predictions = StringIO.StringIO()
    self._predictionLogger.checkpoint(
      checkpointSink=predictions,
      maxRows=int(Configuration.get('nupic.model.checkpoint.maxPredictionRows')))

    self._model.save(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))
    self._jobsDAO.modelSetFields(modelID,
                                 {'modelCheckpointId':str(self._modelCheckpointGUID)},
                                 ignoreUnchanged=True)

    self._logger.info("Checkpointed Hypersearch Model: modelID: %r, "
                      "checkpointID: %r", self._modelID, checkpointID)
    return

Create a checkpoint from the current model and store it in a dir named after checkpoint GUID and finally store the GUID in the Models DB


def __deleteModelCheckpoint(self, modelID):
    """
    Delete the stored checkpoint for the specified modelID. This function is
    called if the current model is now the best model, making the old model's
    checkpoint obsolete

    Parameters:
    -----------------------------------------------------------------------
    modelID:      The modelID for the checkpoint to delete. This is NOT the
                  unique checkpointID
    """

    checkpointID = \
        self._jobsDAO.modelsGetFields(modelID, ['modelCheckpointId'])[0]

    if checkpointID is None:
      return

    try:
      shutil.rmtree(os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))
    except:
      self._logger.warn("Failed to delete model checkpoint %s. "\
                        "Assuming that another worker has already deleted it",
                        checkpointID)
      return

    self._jobsDAO.modelSetFields(modelID,
                                 {'modelCheckpointId':None},
                                 ignoreUnchanged=True)
    return

Delete the stored checkpoint for the specified modelID . This function is called if the current model is now the best model making the old model s checkpoint obsolete


def __getOptimizedMetricLabel(self):
    """ Get the label for the metric being optimized. This function also caches
    the label in the instance variable self._optimizedMetricLabel

    Parameters:
    -----------------------------------------------------------------------
    metricLabels:   A sequence of all the labels being computed for this model

    Returns:        The label for the metric being optmized over
    """
    matchingKeys = matchPatterns([self._optimizeKeyPattern],
                                  self._getMetricLabels())

    if len(matchingKeys) == 0:
      raise Exception("None of the generated metrics match the specified "
                      "optimization pattern: %s. Available metrics are %s" % \
                       (self._optimizeKeyPattern, self._getMetricLabels()))
    elif len(matchingKeys) > 1:
      raise Exception("The specified optimization pattern '%s' matches more "
              "than one metric: %s" % (self._optimizeKeyPattern, matchingKeys))

    return matchingKeys[0]

Get the label for the metric being optimized . This function also caches the label in the instance variable self . _optimizedMetricLabel


def _getFieldStats(self):
    """
    Method which returns a dictionary of field statistics received from the
    input source.

    Returns:

      fieldStats: dict of dicts where the first level is the field name and
        the second level is the statistic. ie. fieldStats['pounds']['min']

    """

    fieldStats = dict()
    fieldNames = self._inputSource.getFieldNames()
    for field in fieldNames:
      curStats = dict()
      curStats['min'] = self._inputSource.getFieldMin(field)
      curStats['max'] = self._inputSource.getFieldMax(field)
      fieldStats[field] = curStats
    return fieldStats

def _getFieldStats ( self ) : fieldStats = dict ( ) fieldNames = self . _inputSource . getFieldNames ( ) for field in fieldNames : curStats = dict ( ) curStats [ 'min' ] = self . _inputSource . getFieldMin ( field ) curStats [ 'max' ] = self . _inputSource . getFieldMax ( field ) fieldStats [ field ] = curStats return fieldStats

Method which returns a dictionary of field statistics received from the input source .


def _updateModelDBResults(self):
    """ Retrieves the current results and updates the model's record in
    the Model database.
    """

    # -----------------------------------------------------------------------
    # Get metrics
    metrics = self._getMetrics()

    # -----------------------------------------------------------------------
    # Extract report metrics that match the requested report REs
    reportDict = dict([(k,metrics[k]) for k in self._reportMetricLabels])

    # -----------------------------------------------------------------------
    # Extract the report item that matches the optimize key RE
    # TODO cache optimizedMetricLabel sooner
    metrics = self._getMetrics()
    optimizeDict = dict()
    if self._optimizeKeyPattern is not None:
      optimizeDict[self._optimizedMetricLabel] = \
                                      metrics[self._optimizedMetricLabel]

    # -----------------------------------------------------------------------
    # Update model results
    results = json.dumps((metrics , optimizeDict))
    self._jobsDAO.modelUpdateResults(self._modelID,  results=results,
                              metricValue=optimizeDict.values()[0],
                              numRecords=(self._currentRecordIndex + 1))

    self._logger.debug(
      "Model Results: modelID=%s; numRecords=%s; results=%s" % \
        (self._modelID, self._currentRecordIndex + 1, results))

    return

Retrieves the current results and updates the model s record in the Model database .


def __checkIfBestCompletedModel(self):
    """
    Reads the current "best model" for the job and returns whether or not the
    current model is better than the "best model" stored for the job

    Returns: (isBetter, storedBest, origResultsStr)

    isBetter:
      True if the current model is better than the stored "best model"
    storedResults:
      A dict of the currently stored results in the jobs table record
    origResultsStr:
      The json-encoded string that currently resides in the "results" field
      of the jobs record (used to create atomicity)
    """

    jobResultsStr = self._jobsDAO.jobGetFields(self._jobID, ['results'])[0]

    if jobResultsStr is None:
        jobResults = {}
    else:
      jobResults = json.loads(jobResultsStr)

    isSaved = jobResults.get('saved', False)
    bestMetric = jobResults.get('bestValue', None)

    currentMetric = self._getMetrics()[self._optimizedMetricLabel]
    self._isBestModel = (not isSaved) \
                        or (currentMetric < bestMetric)

    return self._isBestModel, jobResults, jobResultsStr

def __checkIfBestCompletedModel ( self ) : jobResultsStr = self . _jobsDAO . jobGetFields ( self . _jobID , [ 'results' ] ) [ 0 ] if jobResultsStr is None : jobResults = { } else : jobResults = json . loads ( jobResultsStr ) isSaved = jobResults . get ( 'saved' , False ) bestMetric = jobResults . get ( 'bestValue' , None ) currentMetric = self . _getMetrics ( ) [ self . _optimizedMetricLabel ] self . _isBestModel = ( not isSaved ) or ( currentMetric < bestMetric ) return self . _isBestModel , jobResults , jobResultsStr

Reads the current best model for the job and returns whether or not the current model is better than the best model stored for the job


def _writePrediction(self, result):
    """
    Writes the results of one iteration of a model. The results are written to
    this ModelRunner's in-memory cache unless this model is the "best model" for
    the job. If this model is the "best model", the predictions are written out
    to a permanent store via a prediction output stream instance

    Parameters:
    -----------------------------------------------------------------------
    result:      A opf_utils.ModelResult object, which contains the input and
                  output for this iteration
    """
    self.__predictionCache.append(result)

    if self._isBestModel:
     self.__flushPredictionCache()

def _writePrediction ( self , result ) : self . __predictionCache . append ( result ) if self . _isBestModel : self . __flushPredictionCache ( )

Writes the results of one iteration of a model . The results are written to this ModelRunner s in - memory cache unless this model is the best model for the job . If this model is the best model the predictions are written out to a permanent store via a prediction output stream instance


def __flushPredictionCache(self):
    """
    Writes the contents of this model's in-memory prediction cache to a permanent
    store via the prediction output stream instance
    """

    if not self.__predictionCache:
      return

    # Create an output store, if one doesn't exist already
    if self._predictionLogger is None:
      self._createPredictionLogger()

    startTime = time.time()
    self._predictionLogger.writeRecords(self.__predictionCache,
                                        progressCB=self.__writeRecordsCallback)
    self._logger.info("Flushed prediction cache; numrows=%s; elapsed=%s sec.",
                      len(self.__predictionCache), time.time() - startTime)
    self.__predictionCache.clear()

Writes the contents of this model s in - memory prediction cache to a permanent store via the prediction output stream instance


def __deleteOutputCache(self, modelID):
    """
    Delete's the output cache associated with the given modelID. This actually
    clears up the resources associated with the cache, rather than deleting al
    the records in the cache

    Parameters:
    -----------------------------------------------------------------------
    modelID:      The id of the model whose output cache is being deleted

    """

    # If this is our output, we should close the connection
    if modelID == self._modelID and self._predictionLogger is not None:
      self._predictionLogger.close()
      del self.__predictionCache
      self._predictionLogger = None
      self.__predictionCache = None

Delete s the output cache associated with the given modelID . This actually clears up the resources associated with the cache rather than deleting al the records in the cache


def _initPeriodicActivities(self):
    """ Creates and returns a PeriodicActivityMgr instance initialized with
    our periodic activities

    Parameters:
    -------------------------------------------------------------------------
    retval:             a PeriodicActivityMgr instance
    """

    # Activity to update the metrics for this model
    # in the models table
    updateModelDBResults = PeriodicActivityRequest(repeating=True,
                                                 period=100,
                                                 cb=self._updateModelDBResults)

    updateJobResults = PeriodicActivityRequest(repeating=True,
                                               period=100,
                                               cb=self.__updateJobResultsPeriodic)

    checkCancelation = PeriodicActivityRequest(repeating=True,
                                               period=50,
                                               cb=self.__checkCancelation)

    checkMaturity = PeriodicActivityRequest(repeating=True,
                                            period=10,
                                            cb=self.__checkMaturity)

    # Do an initial update of the job record after 2 iterations to make
    # sure that it is populated with something without having to wait too long
    updateJobResultsFirst = PeriodicActivityRequest(repeating=False,
                                               period=2,
                                               cb=self.__updateJobResultsPeriodic)

    periodicActivities = [updateModelDBResults,
                          updateJobResultsFirst,
                          updateJobResults,
                          checkCancelation]

    if self._isMaturityEnabled:
      periodicActivities.append(checkMaturity)

    return PeriodicActivityMgr(requestedActivities=periodicActivities)


def __checkCancelation(self):
    """ Check if the cancelation flag has been set for this model
    in the Model DB"""

    # Update a hadoop job counter at least once every 600 seconds so it doesn't
    #  think our map task is dead
    print >>sys.stderr, "reporter:counter:HypersearchWorker,numRecords,50"

    # See if the job got cancelled
    jobCancel = self._jobsDAO.jobGetFields(self._jobID, ['cancel'])[0]
    if jobCancel:
      self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED
      self._isCanceled = True
      self._logger.info("Model %s canceled because Job %s was stopped.",
                        self._modelID, self._jobID)
    else:
      stopReason = self._jobsDAO.modelsGetFields(self._modelID, ['engStop'])[0]

      if stopReason is None:
        pass

      elif stopReason == ClientJobsDAO.STOP_REASON_KILLED:
        self._cmpReason = ClientJobsDAO.CMPL_REASON_KILLED
        self._isKilled = True
        self._logger.info("Model %s canceled because it was killed by hypersearch",
                          self._modelID)

      elif stopReason == ClientJobsDAO.STOP_REASON_STOPPED:
        self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED
        self._isCanceled = True
        self._logger.info("Model %s stopped because hypersearch ended", self._modelID)
      else:
        raise RuntimeError ("Unexpected stop reason encountered: %s" % (stopReason))


def __checkMaturity(self):
    """ Save the current metric value and see if the model's performance has
    'leveled off.' We do this by looking at some number of previous number of
    recordings """

    if self._currentRecordIndex+1 < self._MIN_RECORDS_TO_BE_BEST:
      return

    # If we are already mature, don't need to check anything
    if self._isMature:
      return

    metric = self._getMetrics()[self._optimizedMetricLabel]
    self._metricRegression.addPoint(x=self._currentRecordIndex, y=metric)

   # Perform a linear regression to see if the error is leveled off
    #pctChange = self._metricRegression.getPctChange()
    #if pctChange  is not None and abs(pctChange ) <= self._MATURITY_MAX_CHANGE:
    pctChange, absPctChange = self._metricRegression.getPctChanges()
    if pctChange  is not None and absPctChange <= self._MATURITY_MAX_CHANGE:
      self._jobsDAO.modelSetFields(self._modelID,
                                   {'engMatured':True})

      # TODO: Don't stop if we are currently the best model. Also, if we
      # are still running after maturity, we have to periodically check to
      # see if we are still the best model. As soon we lose to some other
      # model, then we should stop at that point.
      self._cmpReason = ClientJobsDAO.CMPL_REASON_STOPPED
      self._isMature = True

      self._logger.info("Model %d has matured (pctChange=%s, n=%d). \n"\
                        "Scores = %s\n"\
                         "Stopping execution",self._modelID, pctChange,
                                              self._MATURITY_NUM_POINTS,
                                              self._metricRegression._window)

Save the current metric value and see if the model s performance has leveled off . We do this by looking at some number of previous number of recordings


def __setAsOrphaned(self):
    """
    Sets the current model as orphaned. This is called when the scheduler is
    about to kill the process to reallocate the worker to a different process.
    """
    cmplReason = ClientJobsDAO.CMPL_REASON_ORPHAN
    cmplMessage = "Killed by Scheduler"
    self._jobsDAO.modelSetCompleted(self._modelID, cmplReason, cmplMessage)

def
__setAsOrphaned
(
self
)
:
cmplReason
=
ClientJobsDAO
.
CMPL_REASON_ORPHAN
cmplMessage
=
"Killed by Scheduler"
self
.
_jobsDAO
.
modelSetCompleted
(
self
.
_modelID
,
cmplReason
,
cmplMessage
)

Sets the current model as orphaned . This is called when the scheduler is about to kill the process to reallocate the worker to a different process .


def readStateFromDB(self):
    """Set our state to that obtained from the engWorkerState field of the
    job record.

    Parameters:
    ---------------------------------------------------------------------
    stateJSON:    JSON encoded state from job record

    """
    self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(self._hsObj._jobID,
                                                    ['engWorkerState'])[0]

    # Init if no prior state yet
    if self._priorStateJSON is None:
      swarms = dict()

      # Fast Swarm, first and only sprint has one swarm for each field
      # in fixedFields
      if self._hsObj._fixedFields is not None:
        print self._hsObj._fixedFields
        encoderSet = []
        for field in self._hsObj._fixedFields:
            if field =='_classifierInput':
              continue
            encoderName = self.getEncoderKeyFromName(field)
            assert encoderName in self._hsObj._encoderNames, "The field '%s' " \
              " specified in the fixedFields list is not present in this " \
              " model." % (field)
            encoderSet.append(encoderName)
        encoderSet.sort()
        swarms['.'.join(encoderSet)] = {
                                'status': 'active',
                                'bestModelId': None,
                                'bestErrScore': None,
                                'sprintIdx': 0,
                                }
      # Temporal prediction search, first sprint has N swarms of 1 field each,
      #  the predicted field may or may not be that one field.
      elif self._hsObj._searchType == HsSearchType.temporal:
        for encoderName in self._hsObj._encoderNames:
          swarms[encoderName] = {
                                  'status': 'active',
                                  'bestModelId': None,
                                  'bestErrScore': None,
                                  'sprintIdx': 0,
                                  }

      # Classification prediction search, first sprint has N swarms of 1 field
      #  each where this field can NOT be the predicted field.
      elif self._hsObj._searchType == HsSearchType.classification:
        for encoderName in self._hsObj._encoderNames:
          if encoderName == self._hsObj._predictedFieldEncoder:
            continue
          swarms[encoderName] = {
                                  'status': 'active',
                                  'bestModelId': None,
                                  'bestErrScore': None,
                                  'sprintIdx': 0,
                                  }

      # Legacy temporal. This is either a model that uses reconstruction or
      #  an older multi-step model that doesn't have a separate
      #  'classifierOnly' encoder for the predicted field. Here, the predicted
      #  field must ALWAYS be present and the first sprint tries the predicted
      #  field only
      elif self._hsObj._searchType == HsSearchType.legacyTemporal:
        swarms[self._hsObj._predictedFieldEncoder] = {
                       'status': 'active',
                       'bestModelId': None,
                       'bestErrScore': None,
                       'sprintIdx': 0,
                       }

      else:
        raise RuntimeError("Unsupported search type: %s" % \
                            (self._hsObj._searchType))

      # Initialize the state.
      self._state = dict(
        # The last time the state was updated by a worker.
        lastUpdateTime = time.time(),

        # Set from within setSwarmState() if we detect that the sprint we just
        #  completed did worse than a prior sprint. This stores the index of
        #  the last good sprint.
        lastGoodSprint = None,

        # Set from within setSwarmState() if lastGoodSprint is True and all
        #  sprints have completed.
        searchOver = False,

        # This is a summary of the active swarms - this information can also
        #  be obtained from the swarms entry that follows, but is summarized here
        #  for easier reference when viewing the state as presented by
        #  log messages and prints of the hsState data structure (by
        #  permutations_runner).
        activeSwarms = swarms.keys(),

        # All the swarms that have been created so far.
        swarms = swarms,

        # All the sprints that have completed or are in progress.
        sprints = [{'status': 'active',
                    'bestModelId': None,
                    'bestErrScore': None}],

        # The list of encoders we have "blacklisted" because they
        #  performed so poorly.
        blackListedEncoders = [],
        )

      # This will do nothing if the value of engWorkerState is not still None.
      self._hsObj._cjDAO.jobSetFieldIfEqual(
          self._hsObj._jobID, 'engWorkerState', json.dumps(self._state), None)

      self._priorStateJSON = self._hsObj._cjDAO.jobGetFields(
          self._hsObj._jobID, ['engWorkerState'])[0]
      assert (self._priorStateJSON is not None)

    # Read state from the database
    self._state = json.loads(self._priorStateJSON)
    self._dirty = False

Set our state to that obtained from the engWorkerState field of the job record .


def getFieldContributions(self):
    """Return the field contributions statistics.

    Parameters:
    ---------------------------------------------------------------------
    retval:   Dictionary where the keys are the field names and the values
                are how much each field contributed to the best score.
    """

    #in the fast swarm, there is only 1 sprint and field contributions are
    #not defined
    if self._hsObj._fixedFields is not None:
      return dict(), dict()
    # Get the predicted field encoder name
    predictedEncoderName = self._hsObj._predictedFieldEncoder

    # -----------------------------------------------------------------------
    # Collect all the single field scores
    fieldScores = []
    for swarmId, info in self._state['swarms'].iteritems():
      encodersUsed = swarmId.split('.')
      if len(encodersUsed) != 1:
        continue
      field = self.getEncoderNameFromKey(encodersUsed[0])
      bestScore = info['bestErrScore']

      # If the bestScore is None, this swarm hasn't completed yet (this could
      #  happen if we're exiting because of maxModels), so look up the best
      #  score so far
      if bestScore is None:
        (_modelId, bestScore) = \
            self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)

      fieldScores.append((bestScore, field))

    # -----------------------------------------------------------------------
    # If we only have 1 field that was tried in the first sprint, then use that
    #  as the base and get the contributions from the fields in the next sprint.
    if self._hsObj._searchType == HsSearchType.legacyTemporal:
      assert(len(fieldScores)==1)
      (baseErrScore, baseField) = fieldScores[0]

      for swarmId, info in self._state['swarms'].iteritems():
        encodersUsed = swarmId.split('.')
        if len(encodersUsed) != 2:
          continue

        fields = [self.getEncoderNameFromKey(name) for name in encodersUsed]
        fields.remove(baseField)

        fieldScores.append((info['bestErrScore'], fields[0]))

    # The first sprint tried a bunch of fields, pick the worst performing one
    #  (within the top self._hsObj._maxBranching ones) as the base
    else:
      fieldScores.sort(reverse=True)

      # If maxBranching was specified, pick the worst performing field within
      #  the top maxBranching+1 fields as our base, which will give that field
      #  a contribution of 0.
      if self._hsObj._maxBranching > 0 \
              and len(fieldScores) > self._hsObj._maxBranching:
        baseErrScore = fieldScores[-self._hsObj._maxBranching-1][0]
      else:
        baseErrScore = fieldScores[0][0]

    # -----------------------------------------------------------------------
    # Prepare and return the fieldContributions dict
    pctFieldContributionsDict = dict()
    absFieldContributionsDict = dict()

    # If we have no base score, can't compute field contributions. This can
    #  happen when we exit early due to maxModels or being cancelled
    if baseErrScore is not None:

      # If the base error score is 0, we can't compute a percent difference
      #  off of it, so move it to a very small float
      if abs(baseErrScore) < 0.00001:
        baseErrScore = 0.00001
      for (errScore, field) in fieldScores:
        if errScore is not None:
          pctBetter = (baseErrScore - errScore) * 100.0 / baseErrScore
        else:
          pctBetter = 0.0
          errScore = baseErrScore   # for absFieldContribution

        pctFieldContributionsDict[field] = pctBetter
        absFieldContributionsDict[field] = baseErrScore - errScore

    self.logger.debug("FieldContributions: %s" % (pctFieldContributionsDict))
    return pctFieldContributionsDict, absFieldContributionsDict

Return the field contributions statistics .


def getAllSwarms(self, sprintIdx):
    """Return the list of all swarms in the given sprint.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids in the given sprint
    """
    swarmIds = []
    for swarmId, info in self._state['swarms'].iteritems():
      if info['sprintIdx'] == sprintIdx:
        swarmIds.append(swarmId)

    return swarmIds

def getAllSwarms ( self , sprintIdx ) : swarmIds = [ ] for swarmId , info in self . _state [ 'swarms' ] . iteritems ( ) : if info [ 'sprintIdx' ] == sprintIdx : swarmIds . append ( swarmId ) return swarmIds

Return the list of all swarms in the given sprint .


def getCompletedSwarms(self):
    """Return the list of all completed swarms.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids
    """
    swarmIds = []
    for swarmId, info in self._state['swarms'].iteritems():
      if info['status'] == 'completed':
        swarmIds.append(swarmId)

    return swarmIds

def getCompletedSwarms ( self ) : swarmIds = [ ] for swarmId , info in self . _state [ 'swarms' ] . iteritems ( ) : if info [ 'status' ] == 'completed' : swarmIds . append ( swarmId ) return swarmIds

Return the list of all completed swarms .


def getCompletingSwarms(self):
    """Return the list of all completing swarms.

    Parameters:
    ---------------------------------------------------------------------
    retval:   list of active swarm Ids
    """
    swarmIds = []
    for swarmId, info in self._state['swarms'].iteritems():
      if info['status'] == 'completing':
        swarmIds.append(swarmId)

    return swarmIds

def getCompletingSwarms ( self ) : swarmIds = [ ] for swarmId , info in self . _state [ 'swarms' ] . iteritems ( ) : if info [ 'status' ] == 'completing' : swarmIds . append ( swarmId ) return swarmIds

Return the list of all completing swarms .


def bestModelInSprint(self, sprintIdx):
    """Return the best model ID and it's errScore from the given sprint,
    which may still be in progress. This returns the best score from all models
    in the sprint which have matured so far.

    Parameters:
    ---------------------------------------------------------------------
    retval:   (modelId, errScore)
    """
    # Get all the swarms in this sprint
    swarms = self.getAllSwarms(sprintIdx)

    # Get the best model and score from each swarm
    bestModelId = None
    bestErrScore = numpy.inf
    for swarmId in swarms:
      (modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)
      if errScore < bestErrScore:
        bestModelId = modelId
        bestErrScore = errScore

    return (bestModelId, bestErrScore)

Return the best model ID and it s errScore from the given sprint which may still be in progress . This returns the best score from all models in the sprint which have matured so far .


def setSwarmState(self, swarmId, newStatus):
    """Change the given swarm's state to 'newState'. If 'newState' is
    'completed', then bestModelId and bestErrScore must be provided.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:      swarm Id
    newStatus:    new status, either 'active', 'completing', 'completed', or
                    'killed'
    """
    assert (newStatus in ['active', 'completing', 'completed', 'killed'])

    # Set the swarm status
    swarmInfo = self._state['swarms'][swarmId]
    if swarmInfo['status'] == newStatus:
      return

    # If some other worker noticed it as completed, setting it to completing
    #  is obviously old information....
    if swarmInfo['status'] == 'completed' and newStatus == 'completing':
      return

    self._dirty = True
    swarmInfo['status'] = newStatus
    if newStatus == 'completed':
      (modelId, errScore) = self._hsObj._resultsDB.bestModelIdAndErrScore(swarmId)
      swarmInfo['bestModelId'] = modelId
      swarmInfo['bestErrScore'] = errScore

    # If no longer active, remove it from the activeSwarms entry
    if newStatus != 'active' and swarmId in self._state['activeSwarms']:
      self._state['activeSwarms'].remove(swarmId)

    # If new status is 'killed', kill off any running particles in that swarm
    if newStatus=='killed':
      self._hsObj.killSwarmParticles(swarmId)

    # In case speculative particles are enabled, make sure we generate a new
    #  swarm at this time if all of the swarms in the current sprint have
    #  completed. This will insure that we don't mark the sprint as completed
    #  before we've created all the possible swarms.
    sprintIdx = swarmInfo['sprintIdx']
    self.isSprintActive(sprintIdx)

    # Update the sprint status. Check all the swarms that belong to this sprint.
    #  If they are all completed, the sprint is completed.
    sprintInfo = self._state['sprints'][sprintIdx]

    statusCounts = dict(active=0, completing=0, completed=0, killed=0)
    bestModelIds = []
    bestErrScores = []
    for info in self._state['swarms'].itervalues():
      if info['sprintIdx'] != sprintIdx:
        continue
      statusCounts[info['status']] += 1
      if info['status'] == 'completed':
        bestModelIds.append(info['bestModelId'])
        bestErrScores.append(info['bestErrScore'])

    if statusCounts['active'] > 0:
      sprintStatus = 'active'
    elif statusCounts['completing'] > 0:
      sprintStatus = 'completing'
    else:
      sprintStatus = 'completed'
    sprintInfo['status'] = sprintStatus

    # If the sprint is complete, get the best model from all of its swarms and
    #  store that as the sprint best
    if sprintStatus == 'completed':
      if len(bestErrScores) > 0:
        whichIdx = numpy.array(bestErrScores).argmin()
        sprintInfo['bestModelId'] = bestModelIds[whichIdx]
        sprintInfo['bestErrScore'] = bestErrScores[whichIdx]
      else:
        # This sprint was empty, most likely because all particles were
        #  killed. Give it a huge error score
        sprintInfo['bestModelId'] = 0
        sprintInfo['bestErrScore'] = numpy.inf

      # See if our best err score got NO BETTER as compared to a previous
      #  sprint. If so, stop exploring subsequent sprints (lastGoodSprint
      #  is no longer None).
      bestPrior = numpy.inf
      for idx in range(sprintIdx):
        if self._state['sprints'][idx]['status'] == 'completed':
          (_, errScore) = self.bestModelInCompletedSprint(idx)
          if errScore is None:
            errScore = numpy.inf
        else:
          errScore = numpy.inf
        if errScore < bestPrior:
          bestPrior = errScore

      if sprintInfo['bestErrScore'] >= bestPrior:
        self._state['lastGoodSprint'] = sprintIdx-1

      # If ALL sprints up to the last good one are done, the search is now over
      if self._state['lastGoodSprint'] is not None \
            and not self.anyGoodSprintsActive():
        self._state['searchOver'] = True

Change the given swarm s state to newState . If newState is completed then bestModelId and bestErrScore must be provided .


def anyGoodSprintsActive(self):
    """Return True if there are any more good sprints still being explored.
    A 'good' sprint is one that is earlier than where we detected an increase
    in error from sprint to subsequent sprint.
    """
    if self._state['lastGoodSprint'] is not None:
      goodSprints = self._state['sprints'][0:self._state['lastGoodSprint']+1]
    else:
      goodSprints = self._state['sprints']

    for sprint in goodSprints:
      if sprint['status'] == 'active':
        anyActiveSprints = True
        break
    else:
      anyActiveSprints = False

    return anyActiveSprints

def anyGoodSprintsActive ( self ) : if self . _state [ 'lastGoodSprint' ] is not None : goodSprints = self . _state [ 'sprints' ] [ 0 : self . _state [ 'lastGoodSprint' ] + 1 ] else : goodSprints = self . _state [ 'sprints' ] for sprint in goodSprints : if sprint [ 'status' ] == 'active' : anyActiveSprints = True break else : anyActiveSprints = False return anyActiveSprints

Return True if there are any more good sprints still being explored . A good sprint is one that is earlier than where we detected an increase in error from sprint to subsequent sprint .


def isSprintCompleted(self, sprintIdx):
    """Return True if the given sprint has completed."""
    numExistingSprints = len(self._state['sprints'])
    if sprintIdx >= numExistingSprints:
      return False

    return (self._state['sprints'][sprintIdx]['status'] == 'completed')

def isSprintCompleted ( self , sprintIdx ) : numExistingSprints = len ( self . _state [ 'sprints' ] ) if sprintIdx >= numExistingSprints : return False return ( self . _state [ 'sprints' ] [ sprintIdx ] [ 'status' ] == 'completed' )

Return True if the given sprint has completed .


def addEncoder(self, name, encoder):
    """
    Adds one encoder.

    :param name: (string) name of encoder, should be unique
    :param encoder: (:class:`.Encoder`) the encoder to add
    """
    self.encoders.append((name, encoder, self.width))
    for d in encoder.getDescription():
      self.description.append((d[0], d[1] + self.width))
    self.width += encoder.getWidth()

def addEncoder ( self , name , encoder ) : self . encoders . append ( ( name , encoder , self . width ) ) for d in encoder . getDescription ( ) : self . description . append ( ( d [ 0 ] , d [ 1 ] + self . width ) ) self . width += encoder . getWidth ( )


def invariant(self):
    """Verify the validity of the node spec object

    The type of each sub-object is verified and then
    the validity of each node spec item is verified by calling
    it invariant() method. It also makes sure that there is at most
    one default input and one default output.
    """
    # Verify the description and singleNodeOnly attributes
    assert isinstance(self.description, str)
    assert isinstance(self.singleNodeOnly, bool)

    # Make sure that all items dicts are really dicts
    assert isinstance(self.inputs, dict)
    assert isinstance(self.outputs, dict)
    assert isinstance(self.parameters, dict)
    assert isinstance(self.commands, dict)

    # Verify all item dicts
    hasDefaultInput = False
    for k, v in self.inputs.items():
      assert isinstance(k, str)
      assert isinstance(v, InputSpec)
      v.invariant()
      if v.isDefaultInput:
        assert not hasDefaultInput
        hasDefaultInput = True

    hasDefaultOutput = False
    for k, v in self.outputs.items():
      assert isinstance(k, str)
      assert isinstance(v, OutputSpec)
      v.invariant()
      if v.isDefaultOutput:
        assert not hasDefaultOutput
        hasDefaultOutput = True

    for k, v in self.parameters.items():
      assert isinstance(k, str)
      assert isinstance(v, ParameterSpec)
      v.invariant()

    for k, v in self.commands.items():
      assert isinstance(k, str)
      assert isinstance(v, CommandSpec)
      v.invariant()


def toDict(self):
    """Convert the information of the node spec to a plain dict of basic types

    The description and singleNodeOnly attributes are placed directly in
    the result dicts. The inputs, outputs, parameters and commands dicts
    contain Spec item objects (InputSpec, OutputSpec, etc). Each such object
    is converted also to a plain dict using the internal items2dict() function
    (see bellow).
    """

    def items2dict(items):
      """Convert a dict of node spec items to a plain dict

      Each node spec item object will be converted to a dict of its
      attributes. The entire items dict will become a dict of dicts (same keys).
      """
      d = {}
      for k, v in items.items():
        d[k] = v.__dict__

      return d

    self.invariant()
    return dict(description=self.description,
                singleNodeOnly=self.singleNodeOnly,
                inputs=items2dict(self.inputs),
                outputs=items2dict(self.outputs),
                parameters=items2dict(self.parameters),
                commands=items2dict(self.commands))


def updateResultsForJob(self, forceUpdate=True):
    """ Chooses the best model for a given job.

    Parameters
    -----------------------------------------------------------------------
    forceUpdate:  (True/False). If True, the update will ignore all the
                  restrictions on the minimum time to update and the minimum
                  number of records to update. This should typically only be
                  set to true if the model has completed running
    """
    updateInterval = time.time() - self._lastUpdateAttemptTime
    if updateInterval < self._MIN_UPDATE_INTERVAL and not forceUpdate:
      return

    self.logger.info("Attempting model selection for jobID=%d: time=%f"\
                     "  lastUpdate=%f"%(self._jobID,
                                        time.time(),
                                        self._lastUpdateAttemptTime))

    timestampUpdated = self._cjDB.jobUpdateSelectionSweep(self._jobID,
                                                          self._MIN_UPDATE_INTERVAL)
    if not timestampUpdated:
      self.logger.info("Unable to update selection sweep timestamp: jobID=%d" \
                       " updateTime=%f"%(self._jobID, self._lastUpdateAttemptTime))
      if not forceUpdate:
        return

    self._lastUpdateAttemptTime = time.time()
    self.logger.info("Succesfully updated selection sweep timestamp jobid=%d updateTime=%f"\
                     %(self._jobID, self._lastUpdateAttemptTime))

    minUpdateRecords = self._MIN_UPDATE_THRESHOLD

    jobResults = self._getJobResults()
    if forceUpdate or jobResults is None:
      minUpdateRecords = 0

    candidateIDs, bestMetric = self._cjDB.modelsGetCandidates(self._jobID, minUpdateRecords)

    self.logger.info("Candidate models=%s, metric=%s, jobID=%s"\
                     %(candidateIDs, bestMetric, self._jobID))

    if len(candidateIDs) == 0:
      return

    self._jobUpdateCandidate(candidateIDs[0], bestMetric, results=jobResults)

Chooses the best model for a given job .


def createEncoder():
  """Create the encoder instance for our test and return it."""
  consumption_encoder = ScalarEncoder(21, 0.0, 100.0, n=50, name="consumption",
      clipInput=True)
  time_encoder = DateEncoder(timeOfDay=(21, 9.5), name="timestamp_timeOfDay")

  encoder = MultiEncoder()
  encoder.addEncoder("consumption", consumption_encoder)
  encoder.addEncoder("timestamp", time_encoder)

  return encoder

def createEncoder ( ) : consumption_encoder = ScalarEncoder ( 21 , 0.0 , 100.0 , n = 50 , name = "consumption" , clipInput = True ) time_encoder = DateEncoder ( timeOfDay = ( 21 , 9.5 ) , name = "timestamp_timeOfDay" ) encoder = MultiEncoder ( ) encoder . addEncoder ( "consumption" , consumption_encoder ) encoder . addEncoder ( "timestamp" , time_encoder ) return encoder

Create the encoder instance for our test and return it .


def __validateExperimentControl(self, control):
    """ Validates control dictionary for the experiment context"""
    # Validate task list
    taskList = control.get('tasks', None)
    if taskList is not None:
      taskLabelsList = []

      for task in taskList:
        validateOpfJsonValue(task, "opfTaskSchema.json")
        validateOpfJsonValue(task['taskControl'], "opfTaskControlSchema.json")

        taskLabel = task['taskLabel']

        assert isinstance(taskLabel, types.StringTypes), \
               "taskLabel type: %r" % type(taskLabel)
        assert len(taskLabel) > 0, "empty string taskLabel not is allowed"

        taskLabelsList.append(taskLabel.lower())

      taskLabelDuplicates = filter(lambda x: taskLabelsList.count(x) > 1,
                                   taskLabelsList)
      assert len(taskLabelDuplicates) == 0, \
             "Duplcate task labels are not allowed: %s" % taskLabelDuplicates

    return


def _matchReportKeys(reportKeyREs=[], allReportKeys=[]):
  """
  Extract all items from the 'allKeys' list whose key matches one of the regular
  expressions passed in 'reportKeys'.

  Parameters:
  ----------------------------------------------------------------------------
  reportKeyREs:     List of regular expressions
  allReportKeys:    List of all keys

  retval:         list of keys from allReportKeys that match the regular expressions
                    in 'reportKeyREs'
                  If an invalid regular expression was included in 'reportKeys',
                    then BadKeyError() is raised
  """

  matchingReportKeys = []

  # Extract the report items of interest
  for keyRE in reportKeyREs:
    # Find all keys that match this regular expression
    matchObj = re.compile(keyRE)
    found = False
    for keyName in allReportKeys:
      match = matchObj.match(keyName)
      if match and match.end() == len(keyName):
        matchingReportKeys.append(keyName)
        found = True
    if not found:
      raise _BadKeyError(keyRE)

  return matchingReportKeys

Extract all items from the allKeys list whose key matches one of the regular expressions passed in reportKeys .


def _getReportItem(itemName, results):
  """
  Get a specific item by name out of the results dict.

  The format of itemName is a string of dictionary keys separated by colons,
  each key being one level deeper into the results dict. For example,
  'key1:key2' would fetch results['key1']['key2'].

  If itemName is not found in results, then None is returned

  """

  subKeys = itemName.split(':')
  subResults = results
  for subKey in subKeys:
    subResults = subResults[subKey]

  return subResults

def _getReportItem ( itemName , results ) : subKeys = itemName . split ( ':' ) subResults = results for subKey in subKeys : subResults = subResults [ subKey ] return subResults

Get a specific item by name out of the results dict .


def _handleModelRunnerException(jobID, modelID, jobsDAO, experimentDir, logger,
                                e):
  """ Perform standard handling of an exception that occurs while running
  a model.

  Parameters:
  -------------------------------------------------------------------------
  jobID:                ID for this hypersearch job in the jobs table
  modelID:              model ID
  jobsDAO:              ClientJobsDAO instance
  experimentDir:        directory containing the experiment
  logger:               the logger to use
  e:                    the exception that occurred
  retval:               (completionReason, completionMsg)
  """

  msg = StringIO.StringIO()
  print >>msg, "Exception occurred while running model %s: %r (%s)" % (
    modelID, e, type(e))
  traceback.print_exc(None, msg)

  completionReason = jobsDAO.CMPL_REASON_ERROR
  completionMsg = msg.getvalue()
  logger.error(completionMsg)

  # Write results to the model database for the error case. Ignore
  # InvalidConnectionException, as this is usually caused by orphaned models
  #
  # TODO: do we really want to set numRecords to 0? Last updated value might
  #       be useful for debugging
  if type(e) is not InvalidConnectionException:
    jobsDAO.modelUpdateResults(modelID,  results=None, numRecords=0)

  # TODO: Make sure this wasn't the best model in job. If so, set the best
  # appropriately

  # If this was an exception that should mark the job as failed, do that
  # now.
  if type(e) == JobFailException:
    workerCmpReason = jobsDAO.jobGetFields(jobID,
        ['workerCompletionReason'])[0]
    if workerCmpReason == ClientJobsDAO.CMPL_REASON_SUCCESS:
      jobsDAO.jobSetFields(jobID, fields=dict(
          cancel=True,
          workerCompletionReason = ClientJobsDAO.CMPL_REASON_ERROR,
          workerCompletionMsg = ": ".join(str(i) for i in e.args)),
          useConnectionID=False,
          ignoreUnchanged=True)

  return (completionReason, completionMsg)

Perform standard handling of an exception that occurs while running a model .


def runModelGivenBaseAndParams(modelID, jobID, baseDescription, params,
            predictedField, reportKeys, optimizeKey, jobsDAO,
            modelCheckpointGUID, logLevel=None, predictionCacheMaxRecords=None):
  """ This creates an experiment directory with a base.py description file
  created from 'baseDescription' and a description.py generated from the
  given params dict and then runs the experiment.

  Parameters:
  -------------------------------------------------------------------------
  modelID:              ID for this model in the models table
  jobID:                ID for this hypersearch job in the jobs table
  baseDescription:      Contents of a description.py with the base experiment
                                          description
  params:               Dictionary of specific parameters to override within
                                  the baseDescriptionFile.
  predictedField:       Name of the input field for which this model is being
                                    optimized
  reportKeys:           Which metrics of the experiment to store into the
                                    results dict of the model's database entry
  optimizeKey:          Which metric we are optimizing for
  jobsDAO               Jobs data access object - the interface to the
                                  jobs database which has the model's table.
  modelCheckpointGUID:  A persistent, globally-unique identifier for
                                  constructing the model checkpoint key
  logLevel:             override logging level to this value, if not None

  retval:               (completionReason, completionMsg)
  """
  from nupic.swarming.ModelRunner import OPFModelRunner

  # The logger for this method
  logger = logging.getLogger('com.numenta.nupic.hypersearch.utils')

  # --------------------------------------------------------------------------
  # Create a temp directory for the experiment and the description files
  experimentDir = tempfile.mkdtemp()
  try:
    logger.info("Using experiment directory: %s" % (experimentDir))

    # Create the decription.py from the overrides in params
    paramsFilePath = os.path.join(experimentDir, 'description.py')
    paramsFile = open(paramsFilePath, 'wb')
    paramsFile.write(_paramsFileHead())

    items = params.items()
    items.sort()
    for (key,value) in items:
      quotedKey = _quoteAndEscape(key)
      if isinstance(value, basestring):

        paramsFile.write("  %s : '%s',\n" % (quotedKey , value))
      else:
        paramsFile.write("  %s : %s,\n" % (quotedKey , value))

    paramsFile.write(_paramsFileTail())
    paramsFile.close()

    # Write out the base description
    baseParamsFile = open(os.path.join(experimentDir, 'base.py'), 'wb')
    baseParamsFile.write(baseDescription)
    baseParamsFile.close()

    # Store the experiment's sub-description file into the model table
    #  for reference
    fd = open(paramsFilePath)
    expDescription = fd.read()
    fd.close()
    jobsDAO.modelSetFields(modelID, {'genDescription': expDescription})

    # Run the experiment now
    try:
      runner = OPFModelRunner(
        modelID=modelID,
        jobID=jobID,
        predictedField=predictedField,
        experimentDir=experimentDir,
        reportKeyPatterns=reportKeys,
        optimizeKeyPattern=optimizeKey,
        jobsDAO=jobsDAO,
        modelCheckpointGUID=modelCheckpointGUID,
        logLevel=logLevel,
        predictionCacheMaxRecords=predictionCacheMaxRecords)

      signal.signal(signal.SIGINT, runner.handleWarningSignal)

      (completionReason, completionMsg) = runner.run()

    except InvalidConnectionException:
      raise
    except Exception, e:

      (completionReason, completionMsg) = _handleModelRunnerException(jobID,
                                     modelID, jobsDAO, experimentDir, logger, e)

  finally:
    # delete our temporary directory tree
    shutil.rmtree(experimentDir)
    signal.signal(signal.SIGINT, signal.default_int_handler)

  # Return completion reason and msg
  return (completionReason, completionMsg)

This creates an experiment directory with a base . py description file created from baseDescription and a description . py generated from the given params dict and then runs the experiment .