def rCopy(d, f=identityConversion, discardNoneKeys=True, deepCopy=True):
  """Recursively copies a dict and returns the result.

  Args:
    d: The dict to copy.
    f: A function to apply to values when copying that takes the value and the
        list of keys from the root of the dict to the value and returns a value
        for the new dict.
    discardNoneKeys: If True, discard key-value pairs when f returns None for
        the value.
    deepCopy: If True, all values in returned dict are true copies (not the
        same object).
  Returns:
    A new dict with keys and values from d replaced with the result of f.
  """
  # Optionally deep copy the dict.
  if deepCopy:
    d = copy.deepcopy(d)

  newDict = {}
  toCopy = [(k, v, newDict, ()) for k, v in d.iteritems()]
  while len(toCopy) > 0:
    k, v, d, prevKeys = toCopy.pop()
    prevKeys = prevKeys + (k,)
    if isinstance(v, dict):
      d[k] = dict()
      toCopy[0:0] = [(innerK, innerV, d[k], prevKeys)
                     for innerK, innerV in v.iteritems()]
    else:
      #print k, v, prevKeys
      newV = f(v, prevKeys)
      if not discardNoneKeys or newV is not None:
        d[k] = newV
  return newDict

def
rCopy
(
d
,
f
=
identityConversion
,
discardNoneKeys
=
True
,
deepCopy
=
True
)
:
# Optionally deep copy the dict.
if
deepCopy
:
d
=
copy
.
deepcopy
(
d
)
newDict
=
{
}
toCopy
=
[
(
k
,
v
,
newDict
,
(
)
)
for
k
,
v
in
d
.
iteritems
(
)
]
while
len
(
toCopy
)
>
0
:
k
,
v
,
d
,
prevKeys
=
toCopy
.
pop
(
)
prevKeys
=
prevKeys
+
(
k
,
)
if
isinstance
(
v
,
dict
)
:
d
[
k
]
=
dict
(
)
toCopy
[
0
:
0
]
=
[
(
innerK
,
innerV
,
d
[
k
]
,
prevKeys
)
for
innerK
,
innerV
in
v
.
iteritems
(
)
]
else
:
#print k, v, prevKeys
newV
=
f
(
v
,
prevKeys
)
if
not
discardNoneKeys
or
newV
is
not
:
d
[
k
]
=
newV
return
newDict

Recursively copies a dict and returns the result .


def rApply(d, f):
  """Recursively applies f to the values in dict d.

  Args:
    d: The dict to recurse over.
    f: A function to apply to values in d that takes the value and a list of
        keys from the root of the dict to the value.
  """
  remainingDicts = [(d, ())]
  while len(remainingDicts) > 0:
    current, prevKeys = remainingDicts.pop()
    for k, v in current.iteritems():
      keys = prevKeys + (k,)
      if isinstance(v, dict):
        remainingDicts.insert(0, (v, keys))
      else:
        f(v, keys)

def rApply ( d , f ) : remainingDicts = [ ( d , ( ) ) ] while len ( remainingDicts ) > 0 : current , prevKeys = remainingDicts . pop ( ) for k , v in current . iteritems ( ) : keys = prevKeys + ( k , ) if isinstance ( v , dict ) : remainingDicts . insert ( 0 , ( v , keys ) ) else : f ( v , keys )

Recursively applies f to the values in dict d .


def clippedObj(obj, maxElementSize=64):
  """
  Return a clipped version of obj suitable for printing, This
  is useful when generating log messages by printing data structures, but
  don't want the message to be too long.

  If passed in a dict, list, or namedtuple, each element of the structure's
  string representation will be limited to 'maxElementSize' characters. This
  will return a new object where the string representation of each element
  has been truncated to fit within maxElementSize.
  """

  # Is it a named tuple?
  if hasattr(obj, '_asdict'):
    obj = obj._asdict()

  # Printing a dict?
  if isinstance(obj, dict):
    objOut = dict()
    for key,val in obj.iteritems():
      objOut[key] = clippedObj(val)

  # Printing a list?
  elif hasattr(obj, '__iter__'):
    objOut = []
    for val in obj:
      objOut.append(clippedObj(val))

  # Some other object
  else:
    objOut = str(obj)
    if len(objOut) > maxElementSize:
      objOut = objOut[0:maxElementSize] + '...'

  return objOut

def
clippedObj
(
obj
,
maxElementSize
=
64
)
:
# Is it a named tuple?
if
hasattr
(
obj
,
'_asdict'
)
:
obj
=
obj
.
_asdict
(
)
# Printing a dict?
if
isinstance
(
obj
,
dict
)
:
objOut
=
dict
(
)
for
key
,
val
in
obj
.
iteritems
(
)
:
objOut
[
key
]
=
clippedObj
(
val
)
# Printing a list?
elif
hasattr
(
obj
,
'__iter__'
)
:
objOut
=
[
]
for
val
in
obj
:
objOut
.
append
(
clippedObj
(
val
)
)
# Some other object
else
:
objOut
=
str
(
obj
)
if
len
(
objOut
)
>
maxElementSize
:
objOut
=
objOut
[
0
:
maxElementSize
]
+
'...'
return
objOut

Return a clipped version of obj suitable for printing This is useful when generating log messages by printing data structures but don t want the message to be too long .


def loadJsonValueFromFile(inputFilePath):
  """ Loads a json value from a file and converts it to the corresponding python
  object.

  inputFilePath:
                  Path of the json file;

  Returns:
                  python value that represents the loaded json value

  """
  with open(inputFilePath) as fileObj:
    value = json.load(fileObj)

  return value

def loadJsonValueFromFile ( inputFilePath ) : with open ( inputFilePath ) as fileObj : value = json . load ( fileObj ) return value

Loads a json value from a file and converts it to the corresponding python object .


def tick(self):
    """ Activity tick handler; services all activities

    Returns:      True if controlling iterator says it's okay to keep going;
                  False to stop
    """

    # Run activities whose time has come
    for act in self.__activities:
      if not act.iteratorHolder[0]:
        continue

      try:
        next(act.iteratorHolder[0])
      except StopIteration:
        act.cb()
        if act.repeating:
          act.iteratorHolder[0] = iter(xrange(act.period))
        else:
          act.iteratorHolder[0] = None

    return True

def
tick
(
self
)
:
# Run activities whose time has come
for
act
in
self
.
__activities
:
if
not
act
.
iteratorHolder
[
0
]
:
continue
try
:
next
(
act
.
iteratorHolder
[
0
]
)
except
StopIteration
:
act
.
cb
(
)
if
act
.
repeating
:
act
.
iteratorHolder
[
0
]
=
iter
(
xrange
(
act
.
period
)
)
else
:
act
.
iteratorHolder
[
0
]
=
return
True

Activity tick handler ; services all activities


def rUpdate(original, updates):
  """Recursively updates the values in original with the values from updates."""
  # Keep a list of the sub-dictionaries that need to be updated to avoid having
  # to use recursion (which could fail for dictionaries with a lot of nesting.
  dictPairs = [(original, updates)]
  while len(dictPairs) > 0:
    original, updates = dictPairs.pop()
    for k, v in updates.iteritems():
      if k in original and isinstance(original[k], dict) and isinstance(v, dict):
        dictPairs.append((original[k], v))
      else:
        original[k] = v

Recursively updates the values in original with the values from updates .


def dictDiffAndReport(da, db):
  """ Compares two python dictionaries at the top level and report differences,
  if any, to stdout

  da:             first dictionary
  db:             second dictionary

  Returns:        The same value as returned by dictDiff() for the given args
  """
  differences = dictDiff(da, db)

  if not differences:
    return differences

  if differences['inAButNotInB']:
    print ">>> inAButNotInB: %s" % differences['inAButNotInB']

  if differences['inBButNotInA']:
    print ">>> inBButNotInA: %s" % differences['inBButNotInA']

  for key in differences['differentValues']:
    print ">>> da[%s] != db[%s]" % (key, key)
    print "da[%s] = %r" % (key, da[key])
    print "db[%s] = %r" % (key, db[key])

  return differences

def
dictDiffAndReport
(
da
,
db
)
:
differences
=
dictDiff
(
da
,
db
)
if
not
differences
:
return
differences
if
differences
[
'inAButNotInB'
]
:
print
">>> inAButNotInB: %s"
%
differences
[
'inAButNotInB'
]
if
differences
[
'inBButNotInA'
]
:
print
">>> inBButNotInA: %s"
%
differences
[
'inBButNotInA'
]
for
key
in
differences
[
'differentValues'
]
:
print
">>> da[%s] != db[%s]"
%
(
key
,
key
)
print
"da[%s] = %r"
%
(
key
,
da
[
key
]
)
print
"db[%s] = %r"
%
(
key
,
db
[
key
]
)
return
differences

Compares two python dictionaries at the top level and report differences if any to stdout


def dictDiff(da, db):
  """ Compares two python dictionaries at the top level and return differences

  da:             first dictionary
  db:             second dictionary

  Returns:        None if dictionaries test equal; otherwise returns a
                  dictionary as follows:
                  {
                    'inAButNotInB':
                        <sequence of keys that are in da but not in db>
                    'inBButNotInA':
                        <sequence of keys that are in db but not in da>
                    'differentValues':
                        <sequence of keys whose corresponding values differ
                         between da and db>
                  }
  """
  different = False

  resultDict = dict()

  resultDict['inAButNotInB'] = set(da) - set(db)
  if resultDict['inAButNotInB']:
    different = True

  resultDict['inBButNotInA'] = set(db) - set(da)
  if resultDict['inBButNotInA']:
    different = True

  resultDict['differentValues'] = []
  for key in (set(da) - resultDict['inAButNotInB']):
    comparisonResult = da[key] == db[key]
    if isinstance(comparisonResult, bool):
      isEqual = comparisonResult
    else:
      # This handles numpy arrays (but only at the top level)
      isEqual = comparisonResult.all()
    if not isEqual:
      resultDict['differentValues'].append(key)
      different = True

  assert (((resultDict['inAButNotInB'] or resultDict['inBButNotInA'] or
          resultDict['differentValues']) and different) or not different)

  return resultDict if different else None


def getSpec(cls):
    """Return the Spec for IdentityRegion.
    """
    spec = {
        "description":IdentityRegion.__doc__,
        "singleNodeOnly":True,
        "inputs":{
          "in":{
            "description":"The input vector.",
            "dataType":"Real32",
            "count":0,
            "required":True,
            "regionLevel":False,
            "isDefaultInput":True,
            "requireSplitterMap":False},
        },
        "outputs":{
          "out":{
            "description":"A copy of the input vector.",
            "dataType":"Real32",
            "count":0,
            "regionLevel":True,
            "isDefaultOutput":True},
        },

        "parameters":{
          "dataWidth":{
            "description":"Size of inputs",
            "accessMode":"Read",
            "dataType":"UInt32",
            "count":1,
            "constraints":""},
        },
    }

    return spec

def
getSpec
(
cls
)
:
spec
=
{
"description"
:
IdentityRegion
.
__doc__
,
"singleNodeOnly"
:
True
,
"inputs"
:
{
"in"
:
{
"description"
:
"The input vector."
,
"dataType"
:
"Real32"
,
"count"
:
0
,
"required"
:
True
,
"regionLevel"
:
False
,
"isDefaultInput"
:
True
,
"requireSplitterMap"
:
False
}
,
}
,
"outputs"
:
{
"out"
:
{
"description"
:
"A copy of the input vector."
,
"dataType"
:
"Real32"
,
"count"
:
0
,
"regionLevel"
:
True
,
"isDefaultOutput"
:
True
}
,
}
,
"parameters"
:
{
"dataWidth"
:
{
"description"
:
"Size of inputs"
,
"accessMode"
:
"Read"
,
"dataType"
:
"UInt32"
,
"count"
:
1
,
"constraints"
:
""
}
,
}
,
}
return
spec


def _setRandomEncoderResolution(minResolution=0.001):
  """
  Given model params, figure out the correct resolution for the
  RandomDistributed encoder. Modifies params in place.
  """
  encoder = (
    model_params.MODEL_PARAMS["modelParams"]["sensorParams"]["encoders"]["value"]
  )

  if encoder["type"] == "RandomDistributedScalarEncoder":
    rangePadding = abs(_INPUT_MAX - _INPUT_MIN) * 0.2
    minValue = _INPUT_MIN - rangePadding
    maxValue = _INPUT_MAX + rangePadding
    resolution = max(minResolution,
                     (maxValue - minValue) / encoder.pop("numBuckets")
                    )
    encoder["resolution"] = resolution

def _setRandomEncoderResolution ( minResolution = 0.001 ) : encoder = ( model_params . MODEL_PARAMS [ "modelParams" ] [ "sensorParams" ] [ "encoders" ] [ "value" ] ) if encoder [ "type" ] == "RandomDistributedScalarEncoder" : rangePadding = abs ( _INPUT_MAX - _INPUT_MIN ) * 0.2 minValue = _INPUT_MIN - rangePadding maxValue = _INPUT_MAX + rangePadding resolution = max ( minResolution , ( maxValue - minValue ) / encoder . pop ( "numBuckets" ) ) encoder [ "resolution" ] = resolution

Given model params figure out the correct resolution for the RandomDistributed encoder . Modifies params in place .


def removeLabels(self, start=None, end=None, labelFilter=None):
    """
    Remove labels from each record with record ROWID in range from
    start to end, noninclusive of end. Removes all records if labelFilter is
    None, otherwise only removes the labels eqaul to labelFilter.

    This will recalculate all points from end to the last record stored in the
    internal cache of this classifier.
    """

    if len(self.saved_states) == 0:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'removeLabels'. Model has no saved records.")

    startID = self.saved_states[0].ROWID

    clippedStart = 0 if start is None else max(0, start - startID)
    clippedEnd = len(self.saved_states) if end is None else \
      max(0, min( len( self.saved_states) , end - startID))

    if clippedEnd <= clippedStart:
      raise HTMPredictionModelInvalidRangeError("Invalid supplied range for "
        "'removeLabels'.", debugInfo={
          'requestRange': {
            'startRecordID': start,
            'endRecordID': end
          },
          'clippedRequestRange': {
            'startRecordID': clippedStart,
            'endRecordID': clippedEnd
          },
          'validRange': {
            'startRecordID': startID,
            'endRecordID': self.saved_states[len(self.saved_states)-1].ROWID
          },
          'numRecordsStored': len(self.saved_states)
        })

    # Remove records within the cache
    recordsToDelete = []
    for state in self.saved_states[clippedStart:clippedEnd]:
      if labelFilter is not None:
        if labelFilter in state.anomalyLabel:
          state.anomalyLabel.remove(labelFilter)
      else:
        state.anomalyLabel = []
      state.setByUser = False
      recordsToDelete.append(state)
    self._deleteRecordsFromKNN(recordsToDelete)

    # Remove records not in cache
    self._deleteRangeFromKNN(start, end)

    # Recompute [clippedEnd, ...)
    for state in self.saved_states[clippedEnd:]:
      self._updateState(state)

    return {'status': 'success'}

Remove labels from each record with record ROWID in range from start to end noninclusive of end . Removes all records if labelFilter is None otherwise only removes the labels eqaul to labelFilter .


def _addRecordToKNN(self, record):
    """
    This method will add the record to the KNN classifier.
    """
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    prototype_idx = classifier.getSelf().getParameter('categoryRecencyList')
    category = self._labelListToCategoryNumber(record.anomalyLabel)

    # If record is already in the classifier, overwrite its labeling
    if record.ROWID in prototype_idx:
      knn.prototypeSetCategory(record.ROWID, category)
      return

    # Learn this pattern in the knn
    pattern = self._getStateAnomalyVector(record)
    rowID = record.ROWID
    knn.learn(pattern, category, rowID=rowID)

This method will add the record to the KNN classifier .


def _deleteRecordsFromKNN(self, recordsToDelete):
    """
    This method will remove the given records from the classifier.

    parameters
    ------------
    recordsToDelete - list of records to delete from the classififier
    """
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    prototype_idx = classifier.getSelf().getParameter('categoryRecencyList')

    idsToDelete = [r.ROWID for r in recordsToDelete if \
      not r.setByUser and r.ROWID in prototype_idx]

    nProtos = knn._numPatterns
    knn.removeIds(idsToDelete)
    assert knn._numPatterns == nProtos - len(idsToDelete)

def _deleteRecordsFromKNN ( self , recordsToDelete ) : classifier = self . htm_prediction_model . _getAnomalyClassifier ( ) knn = classifier . getSelf ( ) . _knn prototype_idx = classifier . getSelf ( ) . getParameter ( 'categoryRecencyList' ) idsToDelete = [ r . ROWID for r in recordsToDelete if not r . setByUser and r . ROWID in prototype_idx ] nProtos = knn . _numPatterns knn . removeIds ( idsToDelete ) assert knn . _numPatterns == nProtos - len ( idsToDelete )

This method will remove the given records from the classifier .


def _deleteRangeFromKNN(self, start=0, end=None):
    """
    This method will remove any stored records within the range from start to
    end. Noninclusive of end.

    parameters
    ------------
    start - integer representing the ROWID of the start of the deletion range,
    end - integer representing the ROWID of the end of the deletion range,
      if None, it will default to end.
    """
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    prototype_idx = numpy.array(
      classifier.getSelf().getParameter('categoryRecencyList'))

    if end is None:
      end = prototype_idx.max() + 1

    idsIdxToDelete = numpy.logical_and(prototype_idx >= start,
                                       prototype_idx < end)
    idsToDelete = prototype_idx[idsIdxToDelete]

    nProtos = knn._numPatterns
    knn.removeIds(idsToDelete.tolist())
    assert knn._numPatterns == nProtos - len(idsToDelete)

def _deleteRangeFromKNN ( self , start = 0 , end = None ) : classifier = self . htm_prediction_model . _getAnomalyClassifier ( ) knn = classifier . getSelf ( ) . _knn prototype_idx = numpy . array ( classifier . getSelf ( ) . getParameter ( 'categoryRecencyList' ) ) if end is None : end = prototype_idx . max ( ) + 1 idsIdxToDelete = numpy . logical_and ( prototype_idx >= start , prototype_idx < end ) idsToDelete = prototype_idx [ idsIdxToDelete ] nProtos = knn . _numPatterns knn . removeIds ( idsToDelete . tolist ( ) ) assert knn . _numPatterns == nProtos - len ( idsToDelete )

This method will remove any stored records within the range from start to end . Noninclusive of end .


def _recomputeRecordFromKNN(self, record):
    """
    return the classified labeling of record
    """
    inputs = {
      "categoryIn": [None],
      "bottomUpIn": self._getStateAnomalyVector(record),
    }

    outputs = {"categoriesOut": numpy.zeros((1,)),
               "bestPrototypeIndices":numpy.zeros((1,)),
               "categoryProbabilitiesOut":numpy.zeros((1,))}

    # Run inference only to capture state before learning
    classifier = self.htm_prediction_model._getAnomalyClassifier()
    knn = classifier.getSelf()._knn

    # Only use points before record to classify and after the wait period.
    classifier_indexes = \
      numpy.array(classifier.getSelf().getParameter('categoryRecencyList'))
    valid_idx = numpy.where(
        (classifier_indexes >= self._autoDetectWaitRecords) &
        (classifier_indexes < record.ROWID)
      )[0].tolist()

    if len(valid_idx) == 0:
      return None

    classifier.setParameter('inferenceMode', True)
    classifier.setParameter('learningMode', False)
    classifier.getSelf().compute(inputs, outputs)
    classifier.setParameter('learningMode', True)

    classifier_distances = classifier.getSelf().getLatestDistances()
    valid_distances = classifier_distances[valid_idx]
    if valid_distances.min() <= self._classificationMaxDist:
      classifier_indexes_prev = classifier_indexes[valid_idx]
      rowID = classifier_indexes_prev[valid_distances.argmin()]
      indexID = numpy.where(classifier_indexes == rowID)[0][0]
      category = classifier.getSelf().getCategoryList()[indexID]
      return category
    return None


def _constructClassificationRecord(self):
    """
    Construct a _HTMClassificationRecord based on the current state of the
    htm_prediction_model of this classifier.

    ***This will look into the internals of the model and may depend on the
    SP, TM, and KNNClassifier***
    """
    model = self.htm_prediction_model
    sp = model._getSPRegion()
    tm = model._getTPRegion()
    tpImp = tm.getSelf()._tfdr

    # Count the number of unpredicted columns
    activeColumns = sp.getOutputData("bottomUpOut").nonzero()[0]
    score = numpy.in1d(activeColumns, self._prevPredictedColumns).sum()
    score = (self._activeColumnCount - score)/float(self._activeColumnCount)

    spSize = sp.getParameter('activeOutputCount')
    tpSize = tm.getParameter('cellsPerColumn') * tm.getParameter('columnCount')

    classificationVector = numpy.array([])

    if self._vectorType == 'tpc':
      # Classification Vector: [---TM Cells---]
      classificationVector = numpy.zeros(tpSize)
      activeCellMatrix = tpImp.getLearnActiveStateT().reshape(tpSize, 1)
      activeCellIdx = numpy.where(activeCellMatrix > 0)[0]
      if activeCellIdx.shape[0] > 0:
        classificationVector[numpy.array(activeCellIdx, dtype=numpy.uint16)] = 1
    elif self._vectorType == 'sp_tpe':
      # Classification Vecotr: [---SP---|---(TM-SP)----]
      classificationVector = numpy.zeros(spSize+spSize)
      if activeColumns.shape[0] > 0:
        classificationVector[activeColumns] = 1.0

      errorColumns = numpy.setdiff1d(self._prevPredictedColumns, activeColumns)
      if errorColumns.shape[0] > 0:
        errorColumnIndexes = ( numpy.array(errorColumns, dtype=numpy.uint16) +
          spSize )
        classificationVector[errorColumnIndexes] = 1.0
    else:
      raise TypeError("Classification vector type must be either 'tpc' or"
        " 'sp_tpe', current value is %s" % (self._vectorType))

    # Store the state for next time step
    numPredictedCols = len(self._prevPredictedColumns)
    predictedColumns = tm.getOutputData("topDownOut").nonzero()[0]
    self._prevPredictedColumns = copy.deepcopy(predictedColumns)

    if self._anomalyVectorLength is None:
      self._anomalyVectorLength = len(classificationVector)

    result = _CLAClassificationRecord(
      ROWID=int(model.getParameter('__numRunCalls') - 1), #__numRunCalls called
        #at beginning of model.run
      anomalyScore=score,
      anomalyVector=classificationVector.nonzero()[0].tolist(),
      anomalyLabel=[]
    )
    return result

Construct a _HTMClassificationRecord based on the current state of the htm_prediction_model of this classifier .


def compute(self):
    """
    Run an iteration of this anomaly classifier
    """
    result = self._constructClassificationRecord()

    # Classify this point after waiting the classification delay
    if result.ROWID >= self._autoDetectWaitRecords:
      self._updateState(result)

    # Save new classification record and keep history as moving window
    self.saved_states.append(result)
    if len(self.saved_states) > self._history_length:
      self.saved_states.pop(0)

    return result


def setAutoDetectWaitRecords(self, waitRecords):
    """
    Sets the autoDetectWaitRecords.
    """
    if not isinstance(waitRecords, int):
      raise HTMPredictionModelInvalidArgument("Invalid argument type \'%s\'. WaitRecord "
        "must be a number." % (type(waitRecords)))

    if len(self.saved_states) > 0 and waitRecords < self.saved_states[0].ROWID:
      raise HTMPredictionModelInvalidArgument("Invalid value. autoDetectWaitRecord value "
        "must be valid record within output stream. Current minimum ROWID in "
        "output stream is %d." % (self.saved_states[0].ROWID))

    self._autoDetectWaitRecords = waitRecords

    # Update all the states in the classifier's cache
    for state in self.saved_states:
      self._updateState(state)


def _allocateSpatialFDR(self, rfInput):
    """Allocate the spatial pooler instance."""
    if self._sfdr:
      return

    # Retrieve the necessary extra arguments that were handled automatically
    autoArgs = dict((name, getattr(self, name))
                     for name in self._spatialArgNames)

    # Instantiate the spatial pooler class.
    if ( (self.SpatialClass == CPPSpatialPooler) or
         (self.SpatialClass == PYSpatialPooler) ):

      autoArgs['columnDimensions'] = [self.columnCount]
      autoArgs['inputDimensions'] = [self.inputWidth]
      autoArgs['potentialRadius'] = self.inputWidth

      self._sfdr = self.SpatialClass(
        **autoArgs
      )


def compute(self, inputs, outputs):
    """
    Run one iteration, profiling it if requested.

    :param inputs: (dict) mapping region input names to numpy.array values
    :param outputs: (dict) mapping region output names to numpy.arrays that
           should be populated with output values by this method
    """

    # Uncomment this to find out who is generating divide by 0, or other numpy warnings
    # numpy.seterr(divide='raise', invalid='raise', over='raise')

    # Modify this line to turn on profiling for a given node. The results file
    #  ('hotshot.stats') will be sensed and printed out by the vision framework's
    #  RunInference.py script at the end of inference.
    # Also uncomment the hotshot import at the top of this file.
    if False and self.learningMode \
        and self._iterations > 0 and self._iterations <= 10:

      import hotshot
      if self._iterations == 10:
        print "\n  Collecting and sorting internal node profiling stats generated by hotshot..."
        stats = hotshot.stats.load("hotshot.stats")
        stats.strip_dirs()
        stats.sort_stats('time', 'calls')
        stats.print_stats()

      # The guts of the compute are contained in the _compute() call so that we
      # can profile it if requested.
      if self._profileObj is None:
        print "\n  Preparing to capture profile using hotshot..."
        if os.path.exists('hotshot.stats'):
          # There is an old hotshot stats profile left over, remove it.
          os.remove('hotshot.stats')
        self._profileObj = hotshot.Profile("hotshot.stats", 1, 1)
                                          # filename, lineevents, linetimings
      self._profileObj.runcall(self._compute, *[inputs, outputs])
    else:
      self._compute(inputs, outputs)

Run one iteration profiling it if requested .


def _compute(self, inputs, outputs):
    """
    Run one iteration of SPRegion's compute
    """

    #if self.topDownMode and (not 'topDownIn' in inputs):
    #  raise RuntimeError("The input topDownIn must be linked in if "
    #                     "topDownMode is True")

    if self._sfdr is None:
      raise RuntimeError("Spatial pooler has not been initialized")

    if not self.topDownMode:
      #
      # BOTTOM-UP compute
      #

      self._iterations += 1

      # Get our inputs into numpy arrays
      buInputVector = inputs['bottomUpIn']

      resetSignal = False
      if 'resetIn' in inputs:
        assert len(inputs['resetIn']) == 1
        resetSignal = inputs['resetIn'][0] != 0

      # Perform inference and/or learning
      rfOutput = self._doBottomUpCompute(
        rfInput = buInputVector.reshape((1,buInputVector.size)),
        resetSignal = resetSignal
        )

      outputs['bottomUpOut'][:] = rfOutput.flat

    else:
      #
      # TOP-DOWN inference
      #

      topDownIn = inputs.get('topDownIn',None)
      spatialTopDownOut, temporalTopDownOut = self._doTopDownInfer(topDownIn)
      outputs['spatialTopDownOut'][:] = spatialTopDownOut
      if temporalTopDownOut is not None:
        outputs['temporalTopDownOut'][:] = temporalTopDownOut

    # OBSOLETE
    outputs['anomalyScore'][:] = 0

#                     "topDownMode is True")


def _initEphemerals(self):
    """
    Initialize all ephemerals used by derived classes.
    """

    if hasattr(self, '_sfdr') and self._sfdr:
      self._spatialPoolerOutput = numpy.zeros(self.columnCount,
                                               dtype=GetNTAReal())
    else:
      self._spatialPoolerOutput = None  # Will be filled in initInNetwork

    # Direct logging support (faster than node watch)
    self._fpLogSPInput = None
    self._fpLogSP = None
    self._fpLogSPDense = None
    self.logPathInput = ""
    self.logPathOutput = ""
    self.logPathOutputDense = ""

Initialize all ephemerals used by derived classes .


def _cacheSequenceInfoType(self):
    """Figure out whether reset, sequenceId,
    both or neither are present in the data.
    Compute once instead of every time.

    Taken from filesource.py"""

    hasReset = self.resetFieldName is not None
    hasSequenceId = self.sequenceIdFieldName is not None

    if hasReset and not hasSequenceId:
      self._sequenceInfoType = self.SEQUENCEINFO_RESET_ONLY
      self._prevSequenceId = 0
    elif not hasReset and hasSequenceId:
      self._sequenceInfoType = self.SEQUENCEINFO_SEQUENCEID_ONLY
      self._prevSequenceId = None
    elif hasReset and hasSequenceId:
      self._sequenceInfoType = self.SEQUENCEINFO_BOTH
    else:
      self._sequenceInfoType = self.SEQUENCEINFO_NONE

def _cacheSequenceInfoType ( self ) : hasReset = self . resetFieldName is not None hasSequenceId = self . sequenceIdFieldName is not None if hasReset and not hasSequenceId : self . _sequenceInfoType = self . SEQUENCEINFO_RESET_ONLY self . _prevSequenceId = 0 elif not hasReset and hasSequenceId : self . _sequenceInfoType = self . SEQUENCEINFO_SEQUENCEID_ONLY self . _prevSequenceId = None elif hasReset and hasSequenceId : self . _sequenceInfoType = self . SEQUENCEINFO_BOTH else : self . _sequenceInfoType = self . SEQUENCEINFO_NONE

Figure out whether reset sequenceId both or neither are present in the data . Compute once instead of every time .


def _getTPClass(temporalImp):
  """ Return the class corresponding to the given temporalImp string
  """

  if temporalImp == 'py':
    return backtracking_tm.BacktrackingTM
  elif temporalImp == 'cpp':
    return backtracking_tm_cpp.BacktrackingTMCPP
  elif temporalImp == 'tm_py':
    return backtracking_tm_shim.TMShim
  elif temporalImp == 'tm_cpp':
    return backtracking_tm_shim.TMCPPShim
  elif temporalImp == 'monitored_tm_py':
    return backtracking_tm_shim.MonitoredTMShim
  else:
    raise RuntimeError("Invalid temporalImp '%s'. Legal values are: 'py', "
              "'cpp', 'tm_py', 'monitored_tm_py'" % (temporalImp))


def _buildArgs(f, self=None, kwargs={}):
  """
  Get the default arguments from the function and assign as instance vars.

  Return a list of 3-tuples with (name, description, defaultValue) for each
    argument to the function.

  Assigns all arguments to the function as instance variables of TMRegion.
  If the argument was not provided, uses the default value.

  Pops any values from kwargs that go to the function.
  """
  # Get the name, description, and default value for each argument
  argTuples = getArgumentDescriptions(f)
  argTuples = argTuples[1:]  # Remove 'self'

  # Get the names of the parameters to our own constructor and remove them
  # Check for _originial_init first, because if LockAttributesMixin is used,
  #  __init__'s signature will be just (self, *args, **kw), but
  #  _original_init is created with the original signature
  #init = getattr(self, '_original_init', self.__init__)
  init = TMRegion.__init__
  ourArgNames = [t[0] for t in getArgumentDescriptions(init)]
  # Also remove a few other names that aren't in our constructor but are
  #  computed automatically (e.g. numberOfCols for the TM)
  ourArgNames += [
    'numberOfCols',    # TM
  ]
  for argTuple in argTuples[:]:
    if argTuple[0] in ourArgNames:
      argTuples.remove(argTuple)

  # Build the dictionary of arguments
  if self:
    for argTuple in argTuples:
      argName = argTuple[0]
      if argName in kwargs:
        # Argument was provided
        argValue = kwargs.pop(argName)
      else:
        # Argument was not provided; use the default value if there is one, and
        #  raise an exception otherwise
        if len(argTuple) == 2:
          # No default value
          raise TypeError("Must provide '%s'" % argName)
        argValue = argTuple[2]
      # Set as an instance variable if 'self' was passed in
      setattr(self, argName, argValue)

  return argTuples

Get the default arguments from the function and assign as instance vars .


def _compute(self, inputs, outputs):
    """
    Run one iteration of TMRegion's compute
    """

    #if self.topDownMode and (not 'topDownIn' in inputs):
    # raise RuntimeError("The input topDownIn must be linked in if "
    #                    "topDownMode is True")

    if self._tfdr is None:
      raise RuntimeError("TM has not been initialized")

    # Conditional compute break
    self._conditionalBreak()

    self._iterations += 1

    # Get our inputs as numpy array
    buInputVector = inputs['bottomUpIn']

    # Handle reset signal
    resetSignal = False
    if 'resetIn' in inputs:
      assert len(inputs['resetIn']) == 1
      if inputs['resetIn'][0] != 0:
        self._tfdr.reset()
        self._sequencePos = 0  # Position within the current sequence

    if self.computePredictedActiveCellIndices:
      prevPredictedState = self._tfdr.getPredictedState().reshape(-1).astype('float32')

    if self.anomalyMode:
      prevPredictedColumns = self._tfdr.topDownCompute().copy().nonzero()[0]

    # Perform inference and/or learning
    tpOutput = self._tfdr.compute(buInputVector, self.learningMode, self.inferenceMode)
    self._sequencePos += 1

    # OR'ing together the cells in each column?
    if self.orColumnOutputs:
      tpOutput= tpOutput.reshape(self.columnCount,
                                     self.cellsPerColumn).max(axis=1)

    # Direct logging of non-zero TM outputs
    if self._fpLogTPOutput:
      output = tpOutput.reshape(-1)
      outputNZ = tpOutput.nonzero()[0]
      outStr = " ".join(["%d" % int(token) for token in outputNZ])
      print >>self._fpLogTPOutput, output.size, outStr

    # Write the bottom up out to our node outputs
    outputs['bottomUpOut'][:] = tpOutput.flat

    if self.topDownMode:
      # Top-down compute
      outputs['topDownOut'][:] = self._tfdr.topDownCompute().copy()

    # Set output for use with anomaly classification region if in anomalyMode
    if self.anomalyMode:
      activeLearnCells = self._tfdr.getLearnActiveStateT()
      size = activeLearnCells.shape[0] * activeLearnCells.shape[1]
      outputs['lrnActiveStateT'][:] = activeLearnCells.reshape(size)

      activeColumns = buInputVector.nonzero()[0]
      outputs['anomalyScore'][:] = anomaly.computeRawAnomalyScore(
        activeColumns, prevPredictedColumns)

    if self.computePredictedActiveCellIndices:
      # Reshape so we are dealing with 1D arrays
      activeState = self._tfdr._getActiveState().reshape(-1).astype('float32')
      activeIndices = numpy.where(activeState != 0)[0]
      predictedIndices= numpy.where(prevPredictedState != 0)[0]
      predictedActiveIndices = numpy.intersect1d(activeIndices, predictedIndices)
      outputs["activeCells"].fill(0)
      outputs["activeCells"][activeIndices] = 1
      outputs["predictedActiveCells"].fill(0)
      outputs["predictedActiveCells"][predictedActiveIndices] = 1

#                    "topDownMode is True")


def finishLearning(self):
    """
    Perform an internal optimization step that speeds up inference if we know
    learning will not be performed anymore. This call may, for example, remove
    all potential inputs to each column.
    """
    if self._tfdr is None:
      raise RuntimeError("Temporal memory has not been initialized")

    if hasattr(self._tfdr, 'finishLearning'):
      self.resetSequenceStates()
      self._tfdr.finishLearning()

Perform an internal optimization step that speeds up inference if we know learning will not be performed anymore . This call may for example remove all potential inputs to each column .


def computeRawAnomalyScore(activeColumns, prevPredictedColumns):
  """Computes the raw anomaly score.

  The raw anomaly score is the fraction of active columns not predicted.

  :param activeColumns: array of active column indices
  :param prevPredictedColumns: array of columns indices predicted in prev step
  :returns: anomaly score 0..1 (float)
  """
  nActiveColumns = len(activeColumns)
  if nActiveColumns > 0:
    # Test whether each element of a 1-D array is also present in a second
    # array. Sum to get the total # of columns that are active and were
    # predicted.
    score = numpy.in1d(activeColumns, prevPredictedColumns).sum()
    # Get the percent of active columns that were NOT predicted, that is
    # our anomaly score.
    score = (nActiveColumns - score) / float(nActiveColumns)
  else:
    # There are no active columns.
    score = 0.0

  return score


def compute(self, activeColumns, predictedColumns,
              inputValue=None, timestamp=None):
    """Compute the anomaly score as the percent of active columns not predicted.

    :param activeColumns: array of active column indices
    :param predictedColumns: array of columns indices predicted in this step
                             (used for anomaly in step T+1)
    :param inputValue: (optional) value of current input to encoders
                                  (eg "cat" for category encoder)
                                  (used in anomaly-likelihood)
    :param timestamp: (optional) date timestamp when the sample occured
                                 (used in anomaly-likelihood)
    :returns: the computed anomaly score; float 0..1
    """
    # Start by computing the raw anomaly score.
    anomalyScore = computeRawAnomalyScore(activeColumns, predictedColumns)

    # Compute final anomaly based on selected mode.
    if self._mode == Anomaly.MODE_PURE:
      score = anomalyScore
    elif self._mode == Anomaly.MODE_LIKELIHOOD:
      if inputValue is None:
        raise ValueError("Selected anomaly mode 'Anomaly.MODE_LIKELIHOOD' "
                 "requires 'inputValue' as parameter to compute() method. ")

      probability = self._likelihood.anomalyProbability(
          inputValue, anomalyScore, timestamp)
      # low likelihood -> hi anomaly
      score = 1 - probability
    elif self._mode == Anomaly.MODE_WEIGHTED:
      probability = self._likelihood.anomalyProbability(
          inputValue, anomalyScore, timestamp)
      score = anomalyScore * (1 - probability)

    # Last, do moving-average if windowSize was specified.
    if self._movingAverage is not None:
      score = self._movingAverage.next(score)

    # apply binary discretization if required
    if self._binaryThreshold is not None:
      if score >= self._binaryThreshold:
        score = 1.0
      else:
        score = 0.0

    return score

Compute the anomaly score as the percent of active columns not predicted .


def addGraph(self, data, position=111, xlabel=None, ylabel=None):
    """ Adds a graph to the plot's figure.

    @param data See matplotlib.Axes.plot documentation.
    @param position A 3-digit number. The first two digits define a 2D grid
            where subplots may be added. The final digit specifies the nth grid
            location for the added subplot
    @param xlabel text to be displayed on the x-axis
    @param ylabel text to be displayed on the y-axis
    """
    ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)
    ax.plot(data)
    plt.draw()

def addGraph ( self , data , position = 111 , xlabel = None , ylabel = None ) : ax = self . _addBase ( position , xlabel = xlabel , ylabel = ylabel ) ax . plot ( data ) plt . draw ( )


def addHistogram(self, data, position=111, xlabel=None, ylabel=None,
                   bins=None):
    """ Adds a histogram to the plot's figure.

    @param data See matplotlib.Axes.hist documentation.
    @param position A 3-digit number. The first two digits define a 2D grid
            where subplots may be added. The final digit specifies the nth grid
            location for the added subplot
    @param xlabel text to be displayed on the x-axis
    @param ylabel text to be displayed on the y-axis
    """
    ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)
    ax.hist(data, bins=bins, color="green", alpha=0.8)
    plt.draw()

def addHistogram ( self , data , position = 111 , xlabel = None , ylabel = None , bins = None ) : ax = self . _addBase ( position , xlabel = xlabel , ylabel = ylabel ) ax . hist ( data , bins = bins , color = "green" , alpha = 0.8 ) plt . draw ( )


def add2DArray(self, data, position=111, xlabel=None, ylabel=None, cmap=None,
                 aspect="auto", interpolation="nearest", name=None):
    """ Adds an image to the plot's figure.

    @param data a 2D array. See matplotlib.Axes.imshow documentation.
    @param position A 3-digit number. The first two digits define a 2D grid
            where subplots may be added. The final digit specifies the nth grid
            location for the added subplot
    @param xlabel text to be displayed on the x-axis
    @param ylabel text to be displayed on the y-axis
    @param cmap color map used in the rendering
    @param aspect how aspect ratio is handled during resize
    @param interpolation interpolation method
    """
    if cmap is None:
      # The default colormodel is an ugly blue-red model.
      cmap = cm.Greys

    ax = self._addBase(position, xlabel=xlabel, ylabel=ylabel)
    ax.imshow(data, cmap=cmap, aspect=aspect, interpolation=interpolation)

    if self._show:
      plt.draw()

    if name is not None:
      if not os.path.exists("log"):
        os.mkdir("log")
      plt.savefig("log/{name}.png".format(name=name), bbox_inches="tight",
                  figsize=(8, 6), dpi=400)


def _addBase(self, position, xlabel=None, ylabel=None):
    """ Adds a subplot to the plot's figure at specified position.

    @param position A 3-digit number. The first two digits define a 2D grid
            where subplots may be added. The final digit specifies the nth grid
            location for the added subplot
    @param xlabel text to be displayed on the x-axis
    @param ylabel text to be displayed on the y-axis
    @returns (matplotlib.Axes) Axes instance
    """
    ax = self._fig.add_subplot(position)
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    return ax

def _addBase ( self , position , xlabel = None , ylabel = None ) : ax = self . _fig . add_subplot ( position ) ax . set_xlabel ( xlabel ) ax . set_ylabel ( ylabel ) return ax

Adds a subplot to the plot s figure at specified position .


def getVersion():
  """
  Get version from local file.
  """
  with open(os.path.join(REPO_DIR, "VERSION"), "r") as versionFile:
    return versionFile.read().strip()

def getVersion ( ) : with open ( os . path . join ( REPO_DIR , "VERSION" ) , "r" ) as versionFile : return versionFile . read ( ) . strip ( )


def nupicBindingsPrereleaseInstalled():
  """
  Make an attempt to determine if a pre-release version of nupic.bindings is
  installed already.

  @return: boolean
  """
  try:
    nupicDistribution = pkg_resources.get_distribution("nupic.bindings")
    if pkg_resources.parse_version(nupicDistribution.version).is_prerelease:
      # A pre-release dev version of nupic.bindings is installed.
      return True
  except pkg_resources.DistributionNotFound:
    pass  # Silently ignore.  The absence of nupic.bindings will be handled by
    # setuptools by default

  return False

Make an attempt to determine if a pre - release version of nupic . bindings is installed already .


def findRequirements():
  """
  Read the requirements.txt file and parse into requirements for setup's
  install_requirements option.
  """
  requirementsPath = os.path.join(REPO_DIR, "requirements.txt")
  requirements = parse_file(requirementsPath)

  if nupicBindingsPrereleaseInstalled():
    # User has a pre-release version of nupic.bindings installed, which is only
    # possible if the user installed and built nupic.bindings from source and
    # it is up to the user to decide when to update nupic.bindings.  We'll
    # quietly remove the entry in requirements.txt so as to not conflate the
    # two.
    requirements = [req for req in requirements if "nupic.bindings" not in req]

  return requirements

Read the requirements . txt file and parse into requirements for setup s install_requirements option .


def _indentLines(str, indentLevels = 1, indentFirstLine=True):
  """ Indent all lines in the given string

  str:          input string
  indentLevels: number of levels of indentation to apply
  indentFirstLine: if False, the 1st line will not be indented

  Returns:      The result string with all lines indented
  """

  indent = _ONE_INDENT * indentLevels

  lines = str.splitlines(True)
  result = ''

  if len(lines) > 0 and not indentFirstLine:
    first = 1
    result += lines[0]
  else:
    first = 0

  for line in lines[first:]:
    result += indent + line

  return result

def _indentLines ( str , indentLevels = 1 , indentFirstLine = True ) : indent = _ONE_INDENT * indentLevels lines = str . splitlines ( True ) result = '' if len ( lines ) > 0 and not indentFirstLine : first = 1 result += lines [ 0 ] else : first = 0 for line in lines [ first : ] : result += indent + line return result


def _generateMetricSpecString(inferenceElement, metric,
                              params=None, field=None,
                              returnLabel=False):
  """ Generates the string representation of a MetricSpec object, and returns
  the metric key associated with the metric.

  Parameters:
  -----------------------------------------------------------------------
  inferenceElement:
    An InferenceElement value that indicates which part of the inference this
    metric is computed on

  metric:
    The type of the metric being computed (e.g. aae, avg_error)

  params:
    A dictionary of parameters for the metric. The keys are the parameter names
    and the values should be the parameter values (e.g. window=200)

  field:
    The name of the field for which this metric is being computed

  returnLabel:
    If True, returns the label of the MetricSpec that was generated
  """

  metricSpecArgs = dict(metric=metric,
                        field=field,
                        params=params,
                        inferenceElement=inferenceElement)

  metricSpecAsString = "MetricSpec(%s)" % \
    ', '.join(['%s=%r' % (item[0],item[1])
              for item in metricSpecArgs.iteritems()])

  if not returnLabel:
    return metricSpecAsString

  spec = MetricSpec(**metricSpecArgs)
  metricLabel = spec.getLabel()
  return metricSpecAsString, metricLabel

def
_generateMetricSpecString
(
inferenceElement
,
metric
,
params
=
,
field
=
,
returnLabel
=
False
)
:
metricSpecArgs
=
dict
(
metric
=
metric
,
field
=
field
,
params
=
params
,
inferenceElement
=
inferenceElement
)
metricSpecAsString
=
"MetricSpec(%s)"
%
', '
.
join
(
[
'%s=%r'
%
(
item
[
0
]
,
item
[
1
]
)
for
item
in
metricSpecArgs
.
iteritems
(
)
]
)
if
not
returnLabel
:
return
metricSpecAsString
spec
=
MetricSpec
(
*
*
metricSpecArgs
)
metricLabel
=
spec
.
getLabel
(
)
return
metricSpecAsString
,
metricLabel

Generates the string representation of a MetricSpec object and returns the metric key associated with the metric .


def _generateFileFromTemplates(templateFileNames, outputFilePath,
                              replacementDict):
  """ Generates a file by applying token replacements to the given template
  file

  templateFileName:
                  A list of template file names; these files are assumed to be in
                  the same directory as the running experiment_generator.py script.
                  ExpGenerator will perform the substitution and concanetate
                  the files in the order they are specified

  outputFilePath: Absolute path of the output file

  replacementDict:
                  A dictionary of token/replacement pairs
  """

  # Find out where we're running from so we know where to find templates
  installPath = os.path.dirname(__file__)
  outputFile = open(outputFilePath, "w")
  outputLines = []
  inputLines = []

  firstFile = True
  for templateFileName in templateFileNames:
    # Separate lines from each file by two blank lines.
    if not firstFile:
      inputLines.extend([os.linesep]*2)
    firstFile = False

    inputFilePath = os.path.join(installPath, templateFileName)
    inputFile = open(inputFilePath)
    inputLines.extend(inputFile.readlines())
    inputFile.close()

  print "Writing ", len(inputLines), "lines..."

  for line in inputLines:
    tempLine = line

    # Enumerate through each key in replacementDict and replace with value
    for k, v in replacementDict.iteritems():
      if v is None:
        v = "None"
      tempLine = re.sub(k, v, tempLine)
    outputFile.write(tempLine)
  outputFile.close()


def _getPropertyValue(schema, propertyName, options):
  """Checks to see if property is specified in 'options'. If not, reads the
  default value from the schema"""

  if propertyName not in options:
    paramsSchema = schema['properties'][propertyName]
    if 'default' in paramsSchema:
      options[propertyName] = paramsSchema['default']
    else:
      options[propertyName] = None

def _getPropertyValue ( schema , propertyName , options ) : if propertyName not in options : paramsSchema = schema [ 'properties' ] [ propertyName ] if 'default' in paramsSchema : options [ propertyName ] = paramsSchema [ 'default' ] else : options [ propertyName ] = None

Checks to see if property is specified in options . If not reads the default value from the schema


def _getExperimentDescriptionSchema():
  """
  Returns the experiment description schema. This implementation loads it in
  from file experimentDescriptionSchema.json.

  Parameters:
  --------------------------------------------------------------------------
  Returns:    returns a dict representing the experiment description schema.
  """
  installPath = os.path.dirname(os.path.abspath(__file__))
  schemaFilePath = os.path.join(installPath, "experimentDescriptionSchema.json")
  return json.loads(open(schemaFilePath, 'r').read())

def _getExperimentDescriptionSchema ( ) : installPath = os . path . dirname ( os . path . abspath ( __file__ ) ) schemaFilePath = os . path . join ( installPath , "experimentDescriptionSchema.json" ) return json . loads ( open ( schemaFilePath , 'r' ) . read ( ) )

Returns the experiment description schema . This implementation loads it in from file experimentDescriptionSchema . json .


def _generateExtraMetricSpecs(options):
  """Generates the non-default metrics specified by the expGenerator params """
  _metricSpecSchema = {'properties': {}}

  results = []
  for metric in options['metrics']:

    for propertyName in _metricSpecSchema['properties'].keys():
      _getPropertyValue(_metricSpecSchema, propertyName, metric)

    specString, label = _generateMetricSpecString(
                                          field=metric['field'],
                                          metric=metric['metric'],
                                          params=metric['params'],
                                          inferenceElement=\
                                                    metric['inferenceElement'],
                                          returnLabel=True)
    if metric['logged']:
      options['loggedMetrics'].append(label)

    results.append(specString)

  return results

def _generateExtraMetricSpecs ( options ) : _metricSpecSchema = { 'properties' : { } } results = [ ] for metric in options [ 'metrics' ] : for propertyName in _metricSpecSchema [ 'properties' ] . keys ( ) : _getPropertyValue ( _metricSpecSchema , propertyName , metric ) specString , label = _generateMetricSpecString ( field = metric [ 'field' ] , metric = metric [ 'metric' ] , params = metric [ 'params' ] , inferenceElement = metric [ 'inferenceElement' ] , returnLabel = True ) if metric [ 'logged' ] : options [ 'loggedMetrics' ] . append ( label ) results . append ( specString ) return results

Generates the non - default metrics specified by the expGenerator params


def _getPredictedField(options):
  """ Gets the predicted field and it's datatype from the options dictionary

  Returns: (predictedFieldName, predictedFieldType)
  """
  if not options['inferenceArgs'] or \
      not options['inferenceArgs']['predictedField']:
    return None, None

  predictedField = options['inferenceArgs']['predictedField']
  predictedFieldInfo = None
  includedFields = options['includedFields']

  for info in includedFields:
    if info['fieldName'] == predictedField:
      predictedFieldInfo = info
      break

  if predictedFieldInfo is None:
    raise ValueError(
      "Predicted field '%s' does not exist in included fields." % predictedField
    )
  predictedFieldType = predictedFieldInfo['fieldType']

  return predictedField, predictedFieldType

Gets the predicted field and it s datatype from the options dictionary


def _generateInferenceArgs(options, tokenReplacements):
  """ Generates the token substitutions related to the predicted field
  and the supplemental arguments for prediction
  """
  inferenceType = options['inferenceType']
  optionInferenceArgs = options.get('inferenceArgs', None)
  resultInferenceArgs = {}
  predictedField = _getPredictedField(options)[0]

  if inferenceType in (InferenceType.TemporalNextStep,
                       InferenceType.TemporalAnomaly):
    assert predictedField,  "Inference Type '%s' needs a predictedField "\
                            "specified in the inferenceArgs dictionary"\
                            % inferenceType

  if optionInferenceArgs:
    # If we will be using a dynamically created predictionSteps, plug in that
    #  variable name in place of the constant scalar value
    if options['dynamicPredictionSteps']:
      altOptionInferenceArgs = copy.deepcopy(optionInferenceArgs)
      altOptionInferenceArgs['predictionSteps'] = '$REPLACE_ME'
      resultInferenceArgs = pprint.pformat(altOptionInferenceArgs)
      resultInferenceArgs = resultInferenceArgs.replace("'$REPLACE_ME'",
                                                        '[predictionSteps]')
    else:
      resultInferenceArgs = pprint.pformat(optionInferenceArgs)

  tokenReplacements['\$INFERENCE_ARGS'] = resultInferenceArgs

  tokenReplacements['\$PREDICTION_FIELD'] = predictedField


def expGenerator(args):
  """ Parses, validates, and executes command-line options;

  On success: Performs requested operation and exits program normally

  On Error:   Dumps exception/error info in JSON format to stdout and exits the
              program with non-zero status.
  """

  # -----------------------------------------------------------------
  # Parse command line options
  #
  parser = OptionParser()
  parser.set_usage("%prog [options] --description='{json object with args}'\n" + \
                   "%prog [options] --descriptionFromFile='{filename}'\n" + \
                   "%prog [options] --showSchema")

  parser.add_option("--description", dest = "description",
    help = "Tells ExpGenerator to generate an experiment description.py and " \
           "permutations.py file using the given JSON formatted experiment "\
           "description string.")

  parser.add_option("--descriptionFromFile", dest = 'descriptionFromFile',
    help = "Tells ExpGenerator to open the given filename and use it's " \
           "contents as the JSON formatted experiment description.")

  parser.add_option("--claDescriptionTemplateFile",
    dest = 'claDescriptionTemplateFile',
    default = 'claDescriptionTemplate.tpl',
    help = "The file containing the template description file for " \
           " ExpGenerator [default: %default]")

  parser.add_option("--showSchema",
                    action="store_true", dest="showSchema",
                    help="Prints the JSON schemas for the --description arg.")

  parser.add_option("--version", dest = 'version', default='v2',
    help = "Generate the permutations file for this version of hypersearch."
            " Possible choices are 'v1' and 'v2' [default: %default].")

  parser.add_option("--outDir",
                    dest = "outDir", default=None,
                    help = "Where to generate experiment. If not specified, " \
                           "then a temp directory will be created"
                    )
  (options, remainingArgs) = parser.parse_args(args)

  #print("OPTIONS=%s" % (str(options)))

  # -----------------------------------------------------------------
  # Check for unprocessed args
  #
  if len(remainingArgs) > 0:
    raise _InvalidCommandArgException(
      _makeUsageErrorStr("Unexpected command-line args: <%s>" % \
                         (' '.join(remainingArgs),), parser.get_usage()))

  # -----------------------------------------------------------------
  # Check for use of mutually-exclusive options
  #
  activeOptions = filter(lambda x: getattr(options, x) != None,
                         ('description', 'showSchema'))
  if len(activeOptions) > 1:
    raise _InvalidCommandArgException(
      _makeUsageErrorStr(("The specified command options are " + \
                          "mutually-exclusive: %s") % (activeOptions,),
                          parser.get_usage()))

  # -----------------------------------------------------------------
  # Process requests
  #
  if options.showSchema:
    _handleShowSchemaOption()

  elif options.description:
    _handleDescriptionOption(options.description, options.outDir,
           parser.get_usage(), hsVersion=options.version,
           claDescriptionTemplateFile = options.claDescriptionTemplateFile)

  elif options.descriptionFromFile:
    _handleDescriptionFromFileOption(options.descriptionFromFile,
          options.outDir, parser.get_usage(), hsVersion=options.version,
          claDescriptionTemplateFile = options.claDescriptionTemplateFile)

  else:
    raise _InvalidCommandArgException(
      _makeUsageErrorStr("Error in validating command options. No option "
                         "provided:\n", parser.get_usage()))

Parses validates and executes command - line options ;


def parseTimestamp(s):
  """
  Parses a textual datetime format and return a Python datetime object.

  The supported format is: ``yyyy-mm-dd h:m:s.ms``

  The time component is optional.

  - hours are 00..23 (no AM/PM)
  - minutes are 00..59
  - seconds are 00..59
  - micro-seconds are 000000..999999

  :param s: (string) input time text
  :return: (datetime.datetime)
  """
  s = s.strip()
  for pattern in DATETIME_FORMATS:
    try:
      return datetime.datetime.strptime(s, pattern)
    except ValueError:
      pass
  raise ValueError('The provided timestamp %s is malformed. The supported '
                   'formats are: [%s]' % (s, ', '.join(DATETIME_FORMATS)))

Parses a textual datetime format and return a Python datetime object .


def parseBool(s):
  """
  String to boolean

  :param s: (string)
  :return: (bool)
  """
  l = s.lower()
  if l in ("true", "t", "1"):
    return True
  if l in ("false", "f", "0"):
    return False
  raise Exception("Unable to convert string '%s' to a boolean value" % s)


def unescape(s):
  """
  Unescapes a string that may contain commas, tabs, newlines and dashes

  Commas are decoded from tabs.

  :param s: (string) to unescape
  :returns: (string) unescaped string
  """
  assert isinstance(s, basestring)
  s = s.replace('\t', ',')
  s = s.replace('\\,', ',')
  s = s.replace('\\n', '\n')
  s = s.replace('\\\\', '\\')

  return s

def unescape ( s ) : assert isinstance ( s , basestring ) s = s . replace ( '\t' , ',' ) s = s . replace ( '\\,' , ',' ) s = s . replace ( '\\n' , '\n' ) s = s . replace ( '\\\\' , '\\' ) return s

Unescapes a string that may contain commas tabs newlines and dashes


def parseSdr(s):
  """
  Parses a string containing only 0's and 1's and return a Python list object.

  :param s: (string) string to parse
  :returns: (list) SDR out
  """
  assert isinstance(s, basestring)
  sdr = [int(c) for c in s if c in ("0", "1")]
  if len(sdr) != len(s):
    raise ValueError("The provided string %s is malformed. The string should "
                     "have only 0's and 1's.")

  return sdr

Parses a string containing only 0 s and 1 s and return a Python list object .


def parseStringList(s):
  """
  Parse a string of space-separated numbers, returning a Python list.

  :param s: (string) to parse
  :returns: (list) binary SDR
  """
  assert isinstance(s, basestring)
  return [int(i) for i in s.split()]

def parseStringList ( s ) : assert isinstance ( s , basestring ) return [ int ( i ) for i in s . split ( ) ]

Parse a string of space - separated numbers returning a Python list .


def coordinatesFromIndex(index, dimensions):
  """
  Translate an index into coordinates, using the given coordinate system.

  Similar to ``numpy.unravel_index``.

  :param index: (int) The index of the point. The coordinates are expressed as a
         single index by using the dimensions as a mixed radix definition. For
         example, in dimensions 42x10, the point [1, 4] is index
         1*420 + 4*10 = 460.

  :param dimensions (list of ints) The coordinate system.

  :returns: (list) of coordinates of length ``len(dimensions)``.
  """
  coordinates = [0] * len(dimensions)

  shifted = index
  for i in xrange(len(dimensions) - 1, 0, -1):
    coordinates[i] = shifted % dimensions[i]
    shifted = shifted / dimensions[i]

  coordinates[0] = shifted

  return coordinates

def coordinatesFromIndex ( index , dimensions ) : coordinates = [ 0 ] * len ( dimensions ) shifted = index for i in xrange ( len ( dimensions ) - 1 , 0 , - 1 ) : coordinates [ i ] = shifted % dimensions [ i ] shifted = shifted / dimensions [ i ] coordinates [ 0 ] = shifted return coordinates

Translate an index into coordinates using the given coordinate system .


def indexFromCoordinates(coordinates, dimensions):
  """
  Translate coordinates into an index, using the given coordinate system.

  Similar to ``numpy.ravel_multi_index``.

  :param coordinates: (list of ints) A list of coordinates of length
         ``dimensions.size()``.

  :param dimensions: (list of ints) The coordinate system.

  :returns: (int) The index of the point. The coordinates are expressed as a
            single index by using the dimensions as a mixed radix definition.
            For example, in dimensions 42x10, the point [1, 4] is index
            1*420 + 4*10 = 460.
  """
  index = 0
  for i, dimension in enumerate(dimensions):
    index *= dimension
    index += coordinates[i]

  return index

def indexFromCoordinates ( coordinates , dimensions ) : index = 0 for i , dimension in enumerate ( dimensions ) : index *= dimension index += coordinates [ i ] return index

Translate coordinates into an index using the given coordinate system .


def neighborhood(centerIndex, radius, dimensions):
  """
  Get the points in the neighborhood of a point.

  A point's neighborhood is the n-dimensional hypercube with sides ranging
  [center - radius, center + radius], inclusive. For example, if there are two
  dimensions and the radius is 3, the neighborhood is 6x6. Neighborhoods are
  truncated when they are near an edge.

  This is designed to be fast. In C++ it's fastest to iterate through neighbors
  one by one, calculating them on-demand rather than creating a list of them.
  But in Python it's faster to build up the whole list in batch via a few calls
  to C code rather than calculating them on-demand with lots of calls to Python
  code.

  :param centerIndex: (int) The index of the point. The coordinates are
         expressed as a single index by using the dimensions as a mixed radix
         definition. For example, in dimensions 42x10, the point [1, 4] is index
         1*420 + 4*10 = 460.

  :param radius: (int) The radius of this neighborhood about the
         ``centerIndex``.

  :param dimensions: (indexable sequence) The dimensions of the world outside
         this neighborhood.

  :returns: (numpy array) The points in the neighborhood, including
            ``centerIndex``.
  """
  centerPosition = coordinatesFromIndex(centerIndex, dimensions)

  intervals = []
  for i, dimension in enumerate(dimensions):
    left = max(0, centerPosition[i] - radius)
    right = min(dimension - 1, centerPosition[i] + radius)
    intervals.append(xrange(left, right + 1))

  coords = numpy.array(list(itertools.product(*intervals)))
  return numpy.ravel_multi_index(coords.T, dimensions)

def neighborhood ( centerIndex , radius , dimensions ) : centerPosition = coordinatesFromIndex ( centerIndex , dimensions ) intervals = [ ] for i , dimension in enumerate ( dimensions ) : left = max ( 0 , centerPosition [ i ] - radius ) right = min ( dimension - 1 , centerPosition [ i ] + radius ) intervals . append ( xrange ( left , right + 1 ) ) coords = numpy . array ( list ( itertools . product ( * intervals ) ) ) return numpy . ravel_multi_index ( coords . T , dimensions )

Get the points in the neighborhood of a point .


def _neighbors(coordinate, radius):
    """
    Returns coordinates around given coordinate, within given radius.
    Includes given coordinate.

    @param coordinate (numpy.array) N-dimensional integer coordinate
    @param radius (int) Radius around `coordinate`

    @return (numpy.array) List of coordinates
    """
    ranges = (xrange(n-radius, n+radius+1) for n in coordinate.tolist())
    return numpy.array(list(itertools.product(*ranges)))

def _neighbors ( coordinate , radius ) : ranges = ( xrange ( n - radius , n + radius + 1 ) for n in coordinate . tolist ( ) ) return numpy . array ( list ( itertools . product ( * ranges ) ) )

Returns coordinates around given coordinate within given radius . Includes given coordinate .


def _topWCoordinates(cls, coordinates, w):
    """
    Returns the top W coordinates by order.

    @param coordinates (numpy.array) A 2D numpy array, where each element
                                     is a coordinate
    @param w (int) Number of top coordinates to return
    @return (numpy.array) A subset of `coordinates`, containing only the
                          top ones by order
    """
    orders = numpy.array([cls._orderForCoordinate(c)
                          for c in coordinates.tolist()])
    indices = numpy.argsort(orders)[-w:]
    return coordinates[indices]

def _topWCoordinates ( cls , coordinates , w ) : orders = numpy . array ( [ cls . _orderForCoordinate ( c ) for c in coordinates . tolist ( ) ] ) indices = numpy . argsort ( orders ) [ - w : ] return coordinates [ indices ]

Returns the top W coordinates by order .


def _hashCoordinate(coordinate):
    """Hash a coordinate to a 64 bit integer."""
    coordinateStr = ",".join(str(v) for v in coordinate)
    # Compute the hash and convert to 64 bit int.
    hash = int(int(hashlib.md5(coordinateStr).hexdigest(), 16) % (2 ** 64))
    return hash


def _orderForCoordinate(cls, coordinate):
    """
    Returns the order for a coordinate.

    @param coordinate (numpy.array) Coordinate
    @return (float) A value in the interval [0, 1), representing the
                    order of the coordinate
    """
    seed = cls._hashCoordinate(coordinate)
    rng = Random(seed)
    return rng.getReal64()

def _orderForCoordinate ( cls , coordinate ) : seed = cls . _hashCoordinate ( coordinate ) rng = Random ( seed ) return rng . getReal64 ( )


def _bitForCoordinate(cls, coordinate, n):
    """
    Maps the coordinate to a bit in the SDR.

    @param coordinate (numpy.array) Coordinate
    @param n (int) The number of available bits in the SDR
    @return (int) The index to a bit in the SDR
    """
    seed = cls._hashCoordinate(coordinate)
    rng = Random(seed)
    return rng.getUInt32(n)

def _bitForCoordinate ( cls , coordinate , n ) : seed = cls . _hashCoordinate ( coordinate ) rng = Random ( seed ) return rng . getUInt32 ( n )

Maps the coordinate to a bit in the SDR .


def binSearch(arr, val):
  """
  Function for running binary search on a sorted list.

  :param arr: (list) a sorted list of integers to search
  :param val: (int)  a integer to search for in the sorted array
  :returns: (int) the index of the element if it is found and -1 otherwise.
  """
  i = bisect_left(arr, val)
  if i != len(arr) and arr[i] == val:
    return i
  return -1

def binSearch ( arr , val ) : i = bisect_left ( arr , val ) if i != len ( arr ) and arr [ i ] == val : return i return - 1

Function for running binary search on a sorted list .


def createSegment(self, cell):
    """
    Adds a new segment on a cell.

    :param cell: (int) Cell index
    :returns: (int) New segment index
    """
    cellData = self._cells[cell]

    if len(self._freeFlatIdxs) > 0:
      flatIdx = self._freeFlatIdxs.pop()
    else:
      flatIdx = self._nextFlatIdx
      self._segmentForFlatIdx.append(None)
      self._nextFlatIdx += 1

    ordinal = self._nextSegmentOrdinal
    self._nextSegmentOrdinal += 1

    segment = Segment(cell, flatIdx, ordinal)
    cellData._segments.append(segment)
    self._segmentForFlatIdx[flatIdx] = segment

    return segment

def createSegment ( self , cell ) : cellData = self . _cells [ cell ] if len ( self . _freeFlatIdxs ) > 0 : flatIdx = self . _freeFlatIdxs . pop ( ) else : flatIdx = self . _nextFlatIdx self . _segmentForFlatIdx . append ( None ) self . _nextFlatIdx += 1 ordinal = self . _nextSegmentOrdinal self . _nextSegmentOrdinal += 1 segment = Segment ( cell , flatIdx , ordinal ) cellData . _segments . append ( segment ) self . _segmentForFlatIdx [ flatIdx ] = segment return segment


def destroySegment(self, segment):
    """
    Destroys a segment.

    :param segment: (:class:`Segment`) representing the segment to be destroyed.
    """
    # Remove the synapses from all data structures outside this Segment.
    for synapse in segment._synapses:
      self._removeSynapseFromPresynapticMap(synapse)
    self._numSynapses -= len(segment._synapses)

    # Remove the segment from the cell's list.
    segments = self._cells[segment.cell]._segments
    i = segments.index(segment)
    del segments[i]

    # Free the flatIdx and remove the final reference so the Segment can be
    # garbage-collected.
    self._freeFlatIdxs.append(segment.flatIdx)
    self._segmentForFlatIdx[segment.flatIdx] = None


def createSynapse(self, segment, presynapticCell, permanence):
    """
    Creates a new synapse on a segment.

    :param segment: (:class:`Segment`) Segment object for synapse to be synapsed
           to.
    :param presynapticCell: (int) Source cell index.
    :param permanence: (float) Initial permanence of synapse.
    :returns: (:class:`Synapse`) created synapse
    """
    idx = len(segment._synapses)
    synapse = Synapse(segment, presynapticCell, permanence,
                      self._nextSynapseOrdinal)
    self._nextSynapseOrdinal += 1
    segment._synapses.add(synapse)

    self._synapsesForPresynapticCell[presynapticCell].add(synapse)

    self._numSynapses += 1

    return synapse

def createSynapse ( self , segment , presynapticCell , permanence ) : idx = len ( segment . _synapses ) synapse = Synapse ( segment , presynapticCell , permanence , self . _nextSynapseOrdinal ) self . _nextSynapseOrdinal += 1 segment . _synapses . add ( synapse ) self . _synapsesForPresynapticCell [ presynapticCell ] . add ( synapse ) self . _numSynapses += 1 return synapse


def destroySynapse(self, synapse):
    """
    Destroys a synapse.

    :param synapse: (:class:`Synapse`) synapse to destroy
    """

    self._numSynapses -= 1

    self._removeSynapseFromPresynapticMap(synapse)

    synapse.segment._synapses.remove(synapse)

def destroySynapse ( self , synapse ) : self . _numSynapses -= 1 self . _removeSynapseFromPresynapticMap ( synapse ) synapse . segment . _synapses . remove ( synapse )


def computeActivity(self, activePresynapticCells, connectedPermanence):
    """
    Compute each segment's number of active synapses for a given input.
    In the returned lists, a segment's active synapse count is stored at index
    ``segment.flatIdx``.

    :param activePresynapticCells: (iter) Active cells.
    :param connectedPermanence: (float) Permanence threshold for a synapse to be
           considered connected

    :returns: (tuple) (``numActiveConnectedSynapsesForSegment`` [list],
                      ``numActivePotentialSynapsesForSegment`` [list])
    """

    numActiveConnectedSynapsesForSegment = [0] * self._nextFlatIdx
    numActivePotentialSynapsesForSegment = [0] * self._nextFlatIdx

    threshold = connectedPermanence - EPSILON

    for cell in activePresynapticCells:
      for synapse in self._synapsesForPresynapticCell[cell]:
        flatIdx = synapse.segment.flatIdx
        numActivePotentialSynapsesForSegment[flatIdx] += 1
        if synapse.permanence > threshold:
          numActiveConnectedSynapsesForSegment[flatIdx] += 1

    return (numActiveConnectedSynapsesForSegment,
            numActivePotentialSynapsesForSegment)

def computeActivity ( self , activePresynapticCells , connectedPermanence ) : numActiveConnectedSynapsesForSegment = [ 0 ] * self . _nextFlatIdx numActivePotentialSynapsesForSegment = [ 0 ] * self . _nextFlatIdx threshold = connectedPermanence - EPSILON for cell in activePresynapticCells : for synapse in self . _synapsesForPresynapticCell [ cell ] : flatIdx = synapse . segment . flatIdx numActivePotentialSynapsesForSegment [ flatIdx ] += 1 if synapse . permanence > threshold : numActiveConnectedSynapsesForSegment [ flatIdx ] += 1 return ( numActiveConnectedSynapsesForSegment , numActivePotentialSynapsesForSegment )

Compute each segment s number of active synapses for a given input . In the returned lists a segment s active synapse count is stored at index segment . flatIdx .


def numSegments(self, cell=None):
    """
    Returns the number of segments.

    :param cell: (int) Optional parameter to get the number of segments on a
           cell.
    :returns: (int) Number of segments on all cells if cell is not specified, or
              on a specific specified cell
    """
    if cell is not None:
      return len(self._cells[cell]._segments)

    return self._nextFlatIdx - len(self._freeFlatIdxs)

def numSegments ( self , cell = None ) : if cell is not None : return len ( self . _cells [ cell ] . _segments ) return self . _nextFlatIdx - len ( self . _freeFlatIdxs )


def read(cls, proto):
    """
    Reads deserialized data from proto object

    :param proto: (DynamicStructBuilder) Proto object

    :returns: (:class:`Connections`) instance
    """
    #pylint: disable=W0212
    protoCells = proto.cells
    connections = cls(len(protoCells))

    for cellIdx, protoCell in enumerate(protoCells):
      protoCell = protoCells[cellIdx]
      protoSegments = protoCell.segments
      connections._cells[cellIdx] = CellData()
      segments = connections._cells[cellIdx]._segments

      for segmentIdx, protoSegment in enumerate(protoSegments):
        segment = Segment(cellIdx, connections._nextFlatIdx,
                          connections._nextSegmentOrdinal)

        segments.append(segment)
        connections._segmentForFlatIdx.append(segment)
        connections._nextFlatIdx += 1
        connections._nextSegmentOrdinal += 1

        synapses = segment._synapses
        protoSynapses = protoSegment.synapses

        for synapseIdx, protoSynapse in enumerate(protoSynapses):
          presynapticCell = protoSynapse.presynapticCell
          synapse = Synapse(segment, presynapticCell, protoSynapse.permanence,
                            ordinal=connections._nextSynapseOrdinal)
          connections._nextSynapseOrdinal += 1
          synapses.add(synapse)
          connections._synapsesForPresynapticCell[presynapticCell].add(synapse)

          connections._numSynapses += 1

    #pylint: enable=W0212
    return connections

def
read
(
cls
,
proto
)
:
#pylint: disable=W0212
protoCells
=
proto
.
cells
connections
=
cls
(
len
(
protoCells
)
)
for
cellIdx
,
protoCell
in
enumerate
(
protoCells
)
:
protoCell
=
protoCells
[
cellIdx
]
protoSegments
=
protoCell
.
segments
connections
.
_cells
[
cellIdx
]
=
CellData
(
)
segments
=
connections
.
_cells
[
cellIdx
]
.
_segments
for
segmentIdx
,
protoSegment
in
enumerate
(
protoSegments
)
:
segment
=
Segment
(
cellIdx
,
connections
.
_nextFlatIdx
,
connections
.
_nextSegmentOrdinal
)
segments
.
append
(
segment
)
connections
.
_segmentForFlatIdx
.
append
(
segment
)
connections
.
_nextFlatIdx
+=
1
connections
.
_nextSegmentOrdinal
+=
1
synapses
=
segment
.
_synapses
protoSynapses
=
protoSegment
.
synapses
for
synapseIdx
,
protoSynapse
in
enumerate
(
protoSynapses
)
:
presynapticCell
=
protoSynapse
.
presynapticCell
synapse
=
Synapse
(
segment
,
presynapticCell
,
protoSynapse
.
permanence
,
ordinal
=
connections
.
_nextSynapseOrdinal
)
connections
.
_nextSynapseOrdinal
+=
1
synapses
.
add
(
synapse
)
connections
.
_synapsesForPresynapticCell
[
presynapticCell
]
.
add
(
synapse
)
connections
.
_numSynapses
+=
1
#pylint: enable=W0212
return
connections


def getString(cls, prop):
    """ Retrieve the requested property as a string. If property does not exist,
    then KeyError will be raised.

    :param prop: (string) name of the property
    :raises: KeyError
    :returns: (string) property value
    """
    if cls._properties is None:
      cls._readStdConfigFiles()

    # Allow configuration properties to be overridden via environment variables
    envValue = os.environ.get("%s%s" % (cls.envPropPrefix,
                                        prop.replace('.', '_')), None)
    if envValue is not None:
      return envValue

    return cls._properties[prop]

Retrieve the requested property as a string . If property does not exist then KeyError will be raised .


def getBool(cls, prop):
    """ Retrieve the requested property and return it as a bool. If property
    does not exist, then KeyError will be raised. If the property value is
    neither 0 nor 1, then ValueError will be raised

    :param prop: (string) name of the property
    :raises: KeyError, ValueError
    :returns: (bool) property value
    """

    value = cls.getInt(prop)

    if value not in (0, 1):
      raise ValueError("Expected 0 or 1, but got %r in config property %s" % (
        value, prop))

    return bool(value)

Retrieve the requested property and return it as a bool . If property does not exist then KeyError will be raised . If the property value is neither 0 nor 1 then ValueError will be raised


def set(cls, prop, value):
    """ Set the value of the given configuration property.

    :param prop: (string) name of the property
    :param value: (object) value to set
    """

    if cls._properties is None:
      cls._readStdConfigFiles()

    cls._properties[prop] = str(value)

def set ( cls , prop , value ) : if cls . _properties is None : cls . _readStdConfigFiles ( ) cls . _properties [ prop ] = str ( value )

Set the value of the given configuration property .


def dict(cls):
    """ Return a dict containing all of the configuration properties

    :returns: (dict) containing all configuration properties.
    """

    if cls._properties is None:
      cls._readStdConfigFiles()

    # Make a copy so we can update any current values obtained from environment
    #  variables
    result = dict(cls._properties)
    keys = os.environ.keys()
    replaceKeys = filter(lambda x: x.startswith(cls.envPropPrefix),
                         keys)
    for envKey in replaceKeys:
      key = envKey[len(cls.envPropPrefix):]
      key = key.replace('_', '.')
      result[key] = os.environ[envKey]

    return result


def readConfigFile(cls, filename, path=None):
    """ Parse the given XML file and store all properties it describes.

    :param filename: (string) name of XML file to parse (no path)
    :param path: (string) path of the XML file. If None, then use the standard
                  configuration search path.
    """
    properties = cls._readConfigFile(filename, path)

    # Create properties dict if necessary
    if cls._properties is None:
      cls._properties = dict()

    for name in properties:
      if 'value' in properties[name]:
        cls._properties[name] = properties[name]['value']

def
readConfigFile
(
cls
,
filename
,
path
=
)
:
properties
=
cls
.
_readConfigFile
(
filename
,
path
)
# Create properties dict if necessary
if
cls
.
_properties
is
:
cls
.
_properties
=
dict
(
)
for
name
in
properties
:
if
'value'
in
properties
[
name
]
:
cls
.
_properties
[
name
]
=
properties
[
name
]
[
'value'
]

Parse the given XML file and store all properties it describes .


def getConfigPaths(cls):
    """ Return the list of paths to search for configuration files.

    :returns: (list) of paths
    """
    configPaths = []
    if cls._configPaths is not None:
      return cls._configPaths

    else:
      if 'NTA_CONF_PATH' in os.environ:
        configVar = os.environ['NTA_CONF_PATH']
        # Return as a list of paths
        configPaths = configVar.split(os.pathsep)

      return configPaths

def
getConfigPaths
(
cls
)
:
configPaths
=
[
]
if
cls
.
_configPaths
is
not
:
return
cls
.
_configPaths
else
:
if
'NTA_CONF_PATH'
in
os
.
environ
:
configVar
=
os
.
environ
[
'NTA_CONF_PATH'
]
# Return as a list of paths
configPaths
=
configVar
.
split
(
os
.
pathsep
)
return
configPaths

Return the list of paths to search for configuration files .


def addNoise(input, noise=0.1, doForeground=True, doBackground=True):
  """
  Add noise to the given input.

  Parameters:
  -----------------------------------------------
  input:         the input to add noise to
  noise:         how much noise to add
  doForeground:  If true, turn off some of the 1 bits in the input
  doBackground:  If true, turn on some of the 0 bits in the input

  """
  if doForeground and doBackground:
    return numpy.abs(input -  (numpy.random.random(input.shape) < noise))
  else:
    if doForeground:
      return numpy.logical_and(input, numpy.random.random(input.shape) > noise)
    if doBackground:
      return numpy.logical_or(input, numpy.random.random(input.shape) < noise)
  return input

def addNoise ( input , noise = 0.1 , doForeground = True , doBackground = True ) : if doForeground and doBackground : return numpy . abs ( input - ( numpy . random . random ( input . shape ) < noise ) ) else : if doForeground : return numpy . logical_and ( input , numpy . random . random ( input . shape ) > noise ) if doBackground : return numpy . logical_or ( input , numpy . random . random ( input . shape ) < noise ) return input


def generateCoincMatrix(nCoinc=10, length=500, activity=50):
  """
  Generate a coincidence matrix. This is used to generate random inputs to the
  temporal learner and to compare the predicted output against.

  It generates a matrix of nCoinc rows, each row has length 'length' and has
  a total of 'activity' bits on.

  Parameters:
  -----------------------------------------------
  nCoinc:        the number of rows to generate
  length:        the length of each row
  activity:      the number of ones to put into each row.

  """

  coincMatrix0 = SM32(int(nCoinc), int(length))
  theOnes = numpy.array([1.0] * activity, dtype=numpy.float32)
  for rowIdx in xrange(nCoinc):
    coinc = numpy.array(random.sample(xrange(length),
                activity), dtype=numpy.uint32)
    coinc.sort()
    coincMatrix0.setRowFromSparse(rowIdx, coinc, theOnes)

  # This is the right code to use, it's faster, but it derails the unit
  # testing of the pooling for now.
  coincMatrix = SM32(int(nCoinc), int(length))
  coincMatrix.initializeWithFixedNNZR(activity)

  return coincMatrix0

Generate a coincidence matrix . This is used to generate random inputs to the temporal learner and to compare the predicted output against .


def generateVectors(numVectors=100, length=500, activity=50):
  """
  Generate a list of random sparse distributed vectors.  This is used to generate
  training vectors to the spatial or temporal learner and to compare the predicted
  output against.

  It generates a list of 'numVectors' elements, each element has length 'length'
  and has a total of 'activity' bits on.

  Parameters:
  -----------------------------------------------
  numVectors:    the number of vectors to generate
  length:        the length of each row
  activity:      the number of ones to put into each row.

  """

  vectors = []
  coinc = numpy.zeros(length, dtype='int32')
  indexList = range(length)

  for i in xrange(numVectors):
      coinc[:] = 0
      coinc[random.sample(indexList, activity)] = 1
      vectors.append(coinc.copy())

  return vectors

def generateVectors ( numVectors = 100 , length = 500 , activity = 50 ) : vectors = [ ] coinc = numpy . zeros ( length , dtype = 'int32' ) indexList = range ( length ) for i in xrange ( numVectors ) : coinc [ : ] = 0 coinc [ random . sample ( indexList , activity ) ] = 1 vectors . append ( coinc . copy ( ) ) return vectors

Generate a list of random sparse distributed vectors . This is used to generate training vectors to the spatial or temporal learner and to compare the predicted output against .


def generateSimpleSequences(nCoinc=10, seqLength=[5,6,7], nSeq=100):
  """
  Generate a set of simple sequences. The elements of the sequences will be
  integers from 0 to 'nCoinc'-1. The length of each sequence will be
  randomly chosen from the 'seqLength' list.

  Parameters:
  -----------------------------------------------
  nCoinc:      the number of elements available to use in the sequences
  seqLength:   a list of possible sequence lengths. The length of each
               sequence will be randomly chosen from here.
  nSeq:        The number of sequences to generate

  retval:      a list of sequences. Each sequence is itself a list
               containing the coincidence indices for that sequence.
  """

  coincList = range(nCoinc)
  seqList  = []

  for i in xrange(nSeq):
    if max(seqLength) <= nCoinc:
      seqList.append(random.sample(coincList, random.choice(seqLength)))
    else:
      len = random.choice(seqLength)
      seq = []
      for x in xrange(len):
        seq.append(random.choice(coincList))
      seqList.append(seq)

  return seqList

def generateSimpleSequences ( nCoinc = 10 , seqLength = [ 5 , 6 , 7 ] , nSeq = 100 ) : coincList = range ( nCoinc ) seqList = [ ] for i in xrange ( nSeq ) : if max ( seqLength ) <= nCoinc : seqList . append ( random . sample ( coincList , random . choice ( seqLength ) ) ) else : len = random . choice ( seqLength ) seq = [ ] for x in xrange ( len ) : seq . append ( random . choice ( coincList ) ) seqList . append ( seq ) return seqList

Generate a set of simple sequences . The elements of the sequences will be integers from 0 to nCoinc - 1 . The length of each sequence will be randomly chosen from the seqLength list .


def generateHubSequences(nCoinc=10, hubs = [2,6], seqLength=[5,6,7], nSeq=100):
  """
  Generate a set of hub sequences. These are sequences which contain a hub
  element in the middle. The elements of the sequences will be integers
  from 0 to 'nCoinc'-1. The hub elements will only appear in the middle of
  each sequence. The length of each sequence will be randomly chosen from the
  'seqLength' list.

  Parameters:
  -----------------------------------------------
  nCoinc:        the number of elements available to use in the sequences
  hubs:          which of the elements will be used as hubs.
  seqLength:     a list of possible sequence lengths. The length of each
                        sequence will be randomly chosen from here.
  nSeq:          The number of sequences to generate

  retval:        a list of sequences. Each sequence is itself a list
                containing the coincidence indices for that sequence.
  """

  coincList = range(nCoinc)
  for hub in hubs:
    coincList.remove(hub)

  seqList = []
  for i in xrange(nSeq):
    length = random.choice(seqLength)-1
    seq = random.sample(coincList,length)
    seq.insert(length//2, random.choice(hubs))
    seqList.append(seq)

  return seqList

def generateHubSequences ( nCoinc = 10 , hubs = [ 2 , 6 ] , seqLength = [ 5 , 6 , 7 ] , nSeq = 100 ) : coincList = range ( nCoinc ) for hub in hubs : coincList . remove ( hub ) seqList = [ ] for i in xrange ( nSeq ) : length = random . choice ( seqLength ) - 1 seq = random . sample ( coincList , length ) seq . insert ( length // 2 , random . choice ( hubs ) ) seqList . append ( seq ) return seqList

Generate a set of hub sequences . These are sequences which contain a hub element in the middle . The elements of the sequences will be integers from 0 to nCoinc - 1 . The hub elements will only appear in the middle of each sequence . The length of each sequence will be randomly chosen from the seqLength list .


def generateSimpleCoincMatrix(nCoinc=10, length=500, activity=50):
  """
  Generate a non overlapping coincidence matrix. This is used to generate random
  inputs to the temporal learner and to compare the predicted output against.

  It generates a matrix of nCoinc rows, each row has length 'length' and has
  a total of 'activity' bits on.

  Parameters:
  -----------------------------------------------
  nCoinc:        the number of rows to generate
  length:        the length of each row
  activity:      the number of ones to put into each row.

  """
  assert nCoinc*activity<=length, "can't generate non-overlapping coincidences"
  coincMatrix = SM32(0, length)
  coinc = numpy.zeros(length, dtype='int32')

  for i in xrange(nCoinc):
      coinc[:] = 0
      coinc[i*activity:(i+1)*activity] = 1
      coincMatrix.addRow(coinc)

  return coincMatrix

Generate a non overlapping coincidence matrix . This is used to generate random inputs to the temporal learner and to compare the predicted output against .


def generateSequences(nPatterns=10, patternLen=500, patternActivity=50,
                    hubs=[2,6],  seqLength=[5,6,7],
                    nSimpleSequences=50,  nHubSequences=50):
  """
  Generate a set of simple and hub sequences. A simple sequence contains
  a randomly chosen set of elements from 0 to 'nCoinc-1'. A hub sequence
  always contains a hub element in the middle of it.

  Parameters:
  -----------------------------------------------
  nPatterns:        the number of patterns to use in the sequences.
  patternLen:       The number of elements in each pattern
  patternActivity:  The number of elements that should be active in
                        each pattern
  hubs:             which of the elements will be used as hubs.
  seqLength:        a list of possible sequence lengths. The length of each
                        sequence will be randomly chosen from here.
  nSimpleSequences: The number of simple sequences to generate
  nHubSequences:    The number of hub sequences to generate

  retval:           (seqList, patterns)
                    seqList: a list of sequences. Each sequence is itself a list
                                  containing the input pattern indices for that sequence.
                    patterns: the input patterns used in the seqList.
  """

  # Create the input patterns
  patterns = generateCoincMatrix(nCoinc=nPatterns, length=patternLen,
              activity=patternActivity)

  # Create the raw sequences
  seqList =  generateSimpleSequences(nCoinc=nPatterns, seqLength=seqLength,
                                    nSeq=nSimpleSequences) + \
             generateHubSequences(nCoinc=nPatterns, hubs=hubs, seqLength=seqLength,
                                  nSeq=nHubSequences)

  # Return results
  return (seqList, patterns)

def
generateSequences
(
nPatterns
=
10
,
patternLen
=
500
,
patternActivity
=
50
,
hubs
=
[
2
,
6
]
,
seqLength
=
[
5
,
6
,
7
]
,
nSimpleSequences
=
50
,
nHubSequences
=
50
)
:
# Create the input patterns
patterns
=
generateCoincMatrix
(
nCoinc
=
nPatterns
,
length
=
patternLen
,
activity
=
patternActivity
)
# Create the raw sequences
seqList
=
generateSimpleSequences
(
nCoinc
=
nPatterns
,
seqLength
=
seqLength
,
nSeq
=
nSimpleSequences
)
+
generateHubSequences
(
nCoinc
=
nPatterns
,
hubs
=
hubs
,
seqLength
=
seqLength
,
nSeq
=
nHubSequences
)
# Return results
return
(
seqList
,
patterns
)

Generate a set of simple and hub sequences . A simple sequence contains a randomly chosen set of elements from 0 to nCoinc - 1 . A hub sequence always contains a hub element in the middle of it .


def sameTMParams(tp1, tp2):
  """Given two TM instances, see if any parameters are different."""
  result = True
  for param in ["numberOfCols", "cellsPerColumn", "initialPerm", "connectedPerm",
                "minThreshold", "newSynapseCount", "permanenceInc", "permanenceDec",
                "permanenceMax", "globalDecay", "activationThreshold",
                "doPooling", "segUpdateValidDuration",
                "burnIn", "pamLength", "maxAge"]:
    if getattr(tp1, param) != getattr(tp2,param):
      print param,"is different"
      print getattr(tp1, param), "vs", getattr(tp2,param)
      result = False
  return result

def
sameTMParams
(
tp1
,
tp2
)
:
result
=
True
for
param
in
[
"numberOfCols"
,
"cellsPerColumn"
,
"initialPerm"
,
"connectedPerm"
,
"minThreshold"
,
"newSynapseCount"
,
"permanenceInc"
,
"permanenceDec"
,
"permanenceMax"
,
"globalDecay"
,
"activationThreshold"
,
"doPooling"
,
"segUpdateValidDuration"
,
"burnIn"
,
"pamLength"
,
"maxAge"
]
:
if
getattr
(
tp1
,
param
)
!=
getattr
(
tp2
,
param
)
:
print
param
,
"is different"
print
getattr
(
tp1
,
param
)
,
"vs"
,
getattr
(
tp2
,
param
)
result
=
False
return
result

Given two TM instances see if any parameters are different .


def sameSegment(seg1, seg2):
  """Return True if seg1 and seg2 are identical, ignoring order of synapses"""
  result = True

  # check sequence segment, total activations etc. In case any are floats,
  # check that they are within 0.001.
  for field in [1, 2, 3, 4, 5, 6]:
    if abs(seg1[0][field] - seg2[0][field]) > 0.001:
      result = False

  # Compare number of synapses
  if len(seg1[1:]) != len(seg2[1:]):
    result = False

  # Now compare synapses, ignoring order of synapses
  for syn in seg2[1:]:
    if syn[2] <= 0:
      print "A synapse with zero permanence encountered"
      result = False
  if result == True:
    for syn in seg1[1:]:
      if syn[2] <= 0:
        print "A synapse with zero permanence encountered"
        result = False
      res = sameSynapse(syn, seg2[1:])
      if res == False:
        result = False

  return result

Return True if seg1 and seg2 are identical ignoring order of synapses


def spDiff(SP1,SP2):
    """
    Function that compares two spatial pooler instances. Compares the
    static variables between the two poolers to make sure that they are equivalent.

    Parameters
    -----------------------------------------
    SP1 first spatial pooler to be compared

    SP2 second spatial pooler to be compared

    To establish equality, this function does the following:

    1.Compares the connected synapse matrices for each coincidence

    2.Compare the potential synapse matrices for each coincidence

    3.Compare the permanence matrices for each coincidence

    4.Compare the firing boosts between the two poolers.

    5.Compare the duty cycles before and after inhibition for both poolers

    """
    if(len(SP1._masterConnectedM)!=len(SP2._masterConnectedM)):
        print "Connected synapse matrices are different sizes"
        return False

    if(len(SP1._masterPotentialM)!=len(SP2._masterPotentialM)):
        print "Potential synapse matrices are different sizes"
        return False

    if(len(SP1._masterPermanenceM)!=len(SP2._masterPermanenceM)):
        print "Permanence matrices are different sizes"
        return False

    #iterate over cells
    for i in range(0,len(SP1._masterConnectedM)):
        #grab the Coincidence Matrices and compare them
        connected1 = SP1._masterConnectedM[i]
        connected2 = SP2._masterConnectedM[i]
        if(connected1!=connected2):
            print "Connected Matrices for cell %d different"  % (i)
            return False
        #grab permanence Matrices and compare them
        permanences1 = SP1._masterPermanenceM[i];
        permanences2 = SP2._masterPermanenceM[i];
        if(permanences1!=permanences2):
            print "Permanence Matrices for cell %d different" % (i)
            return False
        #grab the potential connection Matrices and compare them
        potential1 = SP1._masterPotentialM[i];
        potential2 = SP2._masterPotentialM[i];
        if(potential1!=potential2):
            print "Potential Matrices for cell %d different" % (i)
            return False

    #Check firing boosts
    if(not numpy.array_equal(SP1._firingBoostFactors,SP2._firingBoostFactors)):
        print "Firing boost factors are different between spatial poolers"
        return False

    #Check duty cycles after inhibiton
    if(not numpy.array_equal(SP1._dutyCycleAfterInh,SP2._dutyCycleAfterInh)):
        print "Duty cycles after inhibition are different between spatial poolers"
        return False

    #Check duty cycles before inhibition
    if(not numpy.array_equal(SP1._dutyCycleBeforeInh,SP2._dutyCycleBeforeInh)):
        print "Duty cycles before inhibition are different between spatial poolers"
        return False

    print("Spatial Poolers are equivalent")

    return True

Function that compares two spatial pooler instances . Compares the static variables between the two poolers to make sure that they are equivalent .


def _accumulateFrequencyCounts(values, freqCounts=None):
  """
  Accumulate a list of values 'values' into the frequency counts 'freqCounts',
  and return the updated frequency counts

  For example, if values contained the following: [1,1,3,5,1,3,5], and the initial
  freqCounts was None, then the return value would be:
  [0,3,0,2,0,2]
  which corresponds to how many of each value we saw in the input, i.e. there
  were 0 0's, 3 1's, 0 2's, 2 3's, 0 4's, and 2 5's.

  If freqCounts is not None, the values will be added to the existing counts and
  the length of the frequency Counts will be automatically extended as necessary

  Parameters:
  -----------------------------------------------
  values:         The values to accumulate into the frequency counts
  freqCounts:     Accumulated frequency counts so far, or none
  """

  # How big does our freqCounts vector need to be?
  values = numpy.array(values)
  numEntries = values.max() + 1
  if freqCounts is not None:
    numEntries = max(numEntries, freqCounts.size)

  # Where do we accumulate the results?
  if freqCounts is not None:
    if freqCounts.size != numEntries:
      newCounts = numpy.zeros(numEntries, dtype='int32')
      newCounts[0:freqCounts.size] = freqCounts
    else:
      newCounts = freqCounts
  else:
    newCounts = numpy.zeros(numEntries, dtype='int32')

  # Accumulate the new values
  for v in values:
    newCounts[v] += 1

  return newCounts

Accumulate a list of values values into the frequency counts freqCounts and return the updated frequency counts