def _fillInOnTimes(vector, durations):
  """
  Helper function used by averageOnTimePerTimestep. 'durations' is a vector
  which must be the same len as vector. For each "on" in vector, it fills in
  the corresponding element of duration with the duration of that "on" signal
  up until that time

  Parameters:
  -----------------------------------------------
  vector:     vector of output values over time
  durations:  vector same length as 'vector', initialized to 0's.
              This is filled in with the durations of each 'on" signal.

  Example:
  vector:     11100000001100000000011111100000
  durations:  12300000001200000000012345600000
  """

  # Find where the nonzeros are
  nonzeros = numpy.array(vector).nonzero()[0]

  # Nothing to do if vector is empty
  if len(nonzeros) == 0:
    return

  # Special case of only 1 on bit
  if len(nonzeros) == 1:
    durations[nonzeros[0]] = 1
    return

  # Count the consecutive non-zeros
  prev = nonzeros[0]
  onTime = 1
  onStartIdx = prev
  endIdx = nonzeros[-1]
  for idx in nonzeros[1:]:
    if idx != prev+1:
      # Fill in the durations
      durations[onStartIdx:onStartIdx+onTime] = range(1,onTime+1)
      onTime       = 1
      onStartIdx = idx
    else:
      onTime += 1
    prev = idx

  # Fill in the last one
  durations[onStartIdx:onStartIdx+onTime] = range(1,onTime+1)

def
_fillInOnTimes
(
vector
,
durations
)
:
# Find where the nonzeros are
nonzeros
=
numpy
.
array
(
vector
)
.
nonzero
(
)
[
0
]
# Nothing to do if vector is empty
if
len
(
nonzeros
)
==
0
:
return
# Special case of only 1 on bit
if
len
(
nonzeros
)
==
1
:
durations
[
nonzeros
[
0
]
]
=
1
return
# Count the consecutive non-zeros
prev
=
nonzeros
[
0
]
onTime
=
1
onStartIdx
=
prev
endIdx
=
nonzeros
[
1
]
for
idx
in
nonzeros
[
1
:
]
:
if
idx
!=
prev
+
1
:
# Fill in the durations
durations
[
onStartIdx
:
onStartIdx
+
onTime
]
=
range
(
1
,
onTime
+
1
)
onTime
=
1
onStartIdx
=
idx
else
:
onTime
+=
1
prev
=
idx
# Fill in the last one
durations
[
onStartIdx
:
onStartIdx
+
onTime
]
=
range
(
1
,
onTime
+
1
)

Helper function used by averageOnTimePerTimestep . durations is a vector which must be the same len as vector . For each on in vector it fills in the corresponding element of duration with the duration of that on signal up until that time


def averageOnTimePerTimestep(vectors, numSamples=None):
  """
  Computes the average on-time of the outputs that are on at each time step, and
  then averages this over all time steps.

  This metric is resiliant to the number of outputs that are on at each time
  step. That is, if time step 0 has many more outputs on than time step 100, it
  won't skew the results. This is particularly useful when measuring the
  average on-time of things like the temporal memory output where you might
  have many columns bursting at the start of a sequence - you don't want those
  start of sequence bursts to over-influence the calculated average on-time.

  Parameters:
  -----------------------------------------------
  vectors:          the vectors for which the onTime is calculated. Row 0
                    contains the outputs from time step 0, row 1 from time step
                    1, etc.
  numSamples:       the number of elements for which on-time is calculated.
                    If not specified, then all elements are looked at.

  Returns  (scalar average on-time over all time steps,
            list containing frequency counts of each encountered on-time)

  """

  # Special case given a 1 dimensional vector: it represents a single column
  if vectors.ndim == 1:
    vectors.shape = (-1,1)
  numTimeSteps = len(vectors)
  numElements  = len(vectors[0])

  # How many samples will we look at?
  if numSamples is not None:
    import pdb; pdb.set_trace()   # Test this....
    countOn    = numpy.random.randint(0, numElements, numSamples)
    vectors = vectors[:, countOn]

  # Fill in each non-zero of vectors with the on-time that that output was
  #  on for.
  durations = numpy.zeros(vectors.shape, dtype='int32')
  for col in xrange(vectors.shape[1]):
    _fillInOnTimes(vectors[:,col], durations[:,col])

  # Compute the average on time for each time step
  sums = vectors.sum(axis=1)
  sums.clip(min=1, max=numpy.inf, out=sums)
  avgDurations = durations.sum(axis=1, dtype='float64') / sums
  avgOnTime = avgDurations.sum() / (avgDurations > 0).sum()

  # Generate the frequency counts for each duration
  freqCounts = _accumulateFrequencyCounts(avgDurations)
  return (avgOnTime, freqCounts)

Computes the average on - time of the outputs that are on at each time step and then averages this over all time steps .


def averageOnTime(vectors, numSamples=None):
  """
  Returns the average on-time, averaged over all on-time runs.

  Parameters:
  -----------------------------------------------
  vectors:          the vectors for which the onTime is calculated. Row 0
                    contains the outputs from time step 0, row 1 from time step
                    1, etc.
  numSamples:       the number of elements for which on-time is calculated.
                    If not specified, then all elements are looked at.

  Returns:    (scalar average on-time of all outputs,
               list containing frequency counts of each encountered on-time)

  """

  # Special case given a 1 dimensional vector: it represents a single column
  if vectors.ndim == 1:
    vectors.shape = (-1,1)
  numTimeSteps = len(vectors)
  numElements  = len(vectors[0])

  # How many samples will we look at?
  if numSamples is None:
    numSamples = numElements
    countOn    = range(numElements)
  else:
    countOn    = numpy.random.randint(0, numElements, numSamples)

  # Compute the on-times and accumulate the frequency counts of each on-time
  #  encountered
  sumOfLengths = 0.0
  onTimeFreqCounts = None
  n = 0
  for i in countOn:
    (onTime, segments, durations) = _listOfOnTimesInVec(vectors[:,i])
    if onTime != 0.0:
      sumOfLengths += onTime
      n += segments
      onTimeFreqCounts = _accumulateFrequencyCounts(durations, onTimeFreqCounts)

  # Return the average on time of each element that was on.
  if n > 0:
    return (sumOfLengths/n, onTimeFreqCounts)
  else:
    return (0.0, onTimeFreqCounts)

Returns the average on - time averaged over all on - time runs .


def plotHistogram(freqCounts, title='On-Times Histogram', xLabel='On-Time'):
  """
  This is usually used to display a histogram of the on-times encountered
  in a particular output.

  The freqCounts is a vector containg the frequency counts of each on-time
  (starting at an on-time of 0 and going to an on-time = len(freqCounts)-1)

  The freqCounts are typically generated from the averageOnTimePerTimestep
  or averageOnTime methods of this module.

  Parameters:
  -----------------------------------------------
  freqCounts:       The frequency counts to plot
  title:            Title of the plot

  """

  import pylab
  pylab.ion()
  pylab.figure()
  pylab.bar(numpy.arange(len(freqCounts)) - 0.5, freqCounts)
  pylab.title(title)
  pylab.xlabel(xLabel)

def
plotHistogram
(
freqCounts
,
title
=
'On-Times Histogram'
,
xLabel
=
'On-Time'
)
:
import
pylab
pylab
.
ion
(
)
pylab
.
figure
(
)
pylab
.
bar
(
numpy
.
arange
(
len
(
freqCounts
)
)
0.5
,
freqCounts
)
pylab
.
title
(
title
)
pylab
.
xlabel
(
xLabel
)

This is usually used to display a histogram of the on - times encountered in a particular output .


def populationStability(vectors, numSamples=None):
  """
  Returns the stability for the population averaged over multiple time steps

  Parameters:
  -----------------------------------------------
  vectors:          the vectors for which the stability is calculated
  numSamples        the number of time steps where stability is counted

  At each time step, count the fraction of the active elements which are stable
  from the previous step
  Average all the fraction

  """

  # ----------------------------------------------------------------------
  # Calculate the stability
  numVectors = len(vectors)

  if numSamples is None:
    numSamples = numVectors-1
    countOn = range(numVectors-1)
  else:
    countOn = numpy.random.randint(0, numVectors-1, numSamples)

  sigmap = 0.0
  for i in countOn:
    match = checkMatch(vectors[i], vectors[i+1], sparse=False)
    # Ignore reset vectors (all 0's)
    if match[1] != 0:
      sigmap += float(match[0])/match[1]

  return sigmap / numSamples


def percentOutputsStableOverNTimeSteps(vectors, numSamples=None):
  """
  Returns the percent of the outputs that remain completely stable over
  N time steps.

  Parameters:
  -----------------------------------------------
  vectors:        the vectors for which the stability is calculated
  numSamples:     the number of time steps where stability is counted

  For each window of numSamples, count how many outputs are active during
  the entire window.

  """

  # ----------------------------------------------------------------------
  # Calculate the stability
  totalSamples = len(vectors)
  windowSize = numSamples

  # Process each window
  numWindows = 0
  pctStable = 0

  for wStart in range(0, totalSamples-windowSize+1):
    # Count how many elements are active for the entire time
    data = vectors[wStart:wStart+windowSize]
    outputSums = data.sum(axis=0)
    stableOutputs = (outputSums == windowSize).sum()

    # Accumulated
    samplePctStable = float(stableOutputs) / data[0].sum()
    print samplePctStable
    pctStable += samplePctStable
    numWindows += 1

  # Return percent average over all possible windows
  return float(pctStable) / numWindows

Returns the percent of the outputs that remain completely stable over N time steps .


def computeSaturationLevels(outputs, outputsShape, sparseForm=False):
  """
  Compute the saturation for a continuous level. This breaks the level into
  multiple regions and computes the saturation level for each region.

  Parameters:
  --------------------------------------------
  outputs:      output of the level. If sparseForm is True, this is a list of
                  the non-zeros. If sparseForm is False, it is the dense
                  representation
  outputsShape: The shape of the outputs of the level (height, width)
  retval:       (sat, innerSat):
                  sat: list of the saturation levels of each non-empty
                    region of the level (each 0 -> 1.0)
                  innerSat: list of the saturation level of each non-empty region
                       that is not near an edge (each 0 -> 1.0)

  """

  # Get the outputs into a SparseBinaryMatrix
  if not sparseForm:
    outputs = outputs.reshape(outputsShape)
    spOut = SM32(outputs)
  else:
    if len(outputs) > 0:
      assert (outputs.max() < outputsShape[0] * outputsShape[1])
    spOut = SM32(1, outputsShape[0] * outputsShape[1])
    spOut.setRowFromSparse(0, outputs, [1]*len(outputs))
    spOut.reshape(outputsShape[0], outputsShape[1])

  # Get the activity in each local region using the nNonZerosPerBox method
  # This method takes a list of the end row indices and a list of the end
  #  column indices.
  # We will use regions that are 15x15, which give us about a 1/225 (.4%) resolution
  #  on saturation.
  regionSize = 15
  rows = xrange(regionSize+1, outputsShape[0]+1, regionSize)
  cols = xrange(regionSize+1, outputsShape[1]+1, regionSize)
  regionSums = spOut.nNonZerosPerBox(rows, cols)

  # Get all the nonzeros out - those are our saturation sums
  (locations, values) = regionSums.tolist()
  values /= float(regionSize * regionSize)
  sat = list(values)

  # Now, to compute which are the inner regions, we will only take the ones that
  #  are surrounded by activity above, below, left and right
  innerSat = []
  locationSet = set(locations)
  for (location, value) in itertools.izip(locations, values):
    (row, col) = location
    if (row-1,col) in locationSet and (row, col-1) in locationSet \
      and (row+1, col) in locationSet and (row, col+1) in locationSet:
      innerSat.append(value)

  return (sat, innerSat)

Compute the saturation for a continuous level . This breaks the level into multiple regions and computes the saturation level for each region .


def checkMatch(input, prediction, sparse=True, verbosity=0):
  """
  Compares the actual input with the predicted input and returns results

  Parameters:
  -----------------------------------------------
  input:          The actual input
  prediction:     the predicted input
  verbosity:        If > 0, print debugging messages
  sparse:         If true, they are in sparse form (list of
                     active indices)

  retval         (foundInInput, totalActiveInInput, missingFromInput,
                            totalActiveInPrediction)
    foundInInput:       The number of predicted active elements that were
                        found in the actual input
    totalActiveInInput: The total number of active elements in the input.
    missingFromInput:   The number of predicted active elements that were not
                        found in the actual input
    totalActiveInPrediction:  The total number of active elements in the prediction

  """

  if sparse:
    activeElementsInInput = set(input)
    activeElementsInPrediction = set(prediction)

  else:
    activeElementsInInput = set(input.nonzero()[0])
    activeElementsInPrediction = set(prediction.nonzero()[0])

  totalActiveInPrediction = len(activeElementsInPrediction)
  totalActiveInInput     = len(activeElementsInInput)

  foundInInput = len(activeElementsInPrediction.intersection(activeElementsInInput))
  missingFromInput = len(activeElementsInPrediction.difference(activeElementsInInput))
  missingFromPrediction = len(activeElementsInInput.difference(activeElementsInPrediction))

  if verbosity >= 1:
    print "preds. found in input:", foundInInput, "out of", totalActiveInPrediction,
    print "; preds. missing from input:", missingFromInput, "out of", \
              totalActiveInPrediction,
    print "; unexpected active in input:", missingFromPrediction, "out of", \
              totalActiveInInput

  return (foundInInput, totalActiveInInput, missingFromInput,
          totalActiveInPrediction)

def
checkMatch
(
input
,
prediction
,
sparse
=
True
,
verbosity
=
0
)
:
if
sparse
:
activeElementsInInput
=
set
(
input
)
activeElementsInPrediction
=
set
(
prediction
)
else
:
activeElementsInInput
=
set
(
input
.
nonzero
(
)
[
0
]
)
activeElementsInPrediction
=
set
(
prediction
.
nonzero
(
)
[
0
]
)
totalActiveInPrediction
=
len
(
activeElementsInPrediction
)
totalActiveInInput
=
len
(
activeElementsInInput
)
foundInInput
=
len
(
activeElementsInPrediction
.
intersection
(
activeElementsInInput
)
)
missingFromInput
=
len
(
activeElementsInPrediction
.
difference
(
activeElementsInInput
)
)
missingFromPrediction
=
len
(
activeElementsInInput
.
difference
(
activeElementsInPrediction
)
)
if
verbosity
>=
1
:
print
"preds. found in input:"
,
foundInInput
,
"out of"
,
totalActiveInPrediction
,
print
"; preds. missing from input:"
,
missingFromInput
,
"out of"
,
totalActiveInPrediction
,
print
"; unexpected active in input:"
,
missingFromPrediction
,
"out of"
,
totalActiveInInput
return
(
foundInInput
,
totalActiveInInput
,
missingFromInput
,
totalActiveInPrediction
)


def getCentreAndSpreadOffsets(spaceShape,
                              spreadShape,
                              stepSize=1):
  """
  Generates centre offsets and spread offsets for block-mode based training
  regimes - star, cross, block.

    Parameters:
    -----------------------------------------------
    spaceShape:   The (height, width) of the 2-D space to explore. This
                  sets the number of center-points.
    spreadShape:  The shape (height, width) of the area around each center-point
                  to explore.
    stepSize:     The step size. How big each step is, in pixels. This controls
                  *both* the spacing of the center-points within the block and the
                  points we explore around each center-point
    retval:       (centreOffsets, spreadOffsets)
  """

  from nupic.math.cross import cross
  # =====================================================================
  # Init data structures
  # What is the range on the X and Y offsets of the center points?
  shape = spaceShape
  # If the shape is (1,1), special case of just 1 center point
  if shape[0] == 1 and shape[1] == 1:
    centerOffsets = [(0,0)]
  else:
    xMin = -1 * (shape[1] // 2)
    xMax = xMin + shape[1] - 1
    xPositions = range(stepSize * xMin, stepSize * xMax + 1, stepSize)

    yMin = -1 * (shape[0] // 2)
    yMax = yMin + shape[0] - 1
    yPositions = range(stepSize * yMin, stepSize * yMax + 1, stepSize)

    centerOffsets = list(cross(yPositions, xPositions))

  numCenterOffsets = len(centerOffsets)
  print "centerOffsets:", centerOffsets

  # What is the range on the X and Y offsets of the spread points?
  shape = spreadShape
  # If the shape is (1,1), special case of no spreading around each center
  #  point
  if shape[0] == 1 and shape[1] == 1:
    spreadOffsets = [(0,0)]
  else:
    xMin = -1 * (shape[1] // 2)
    xMax = xMin + shape[1] - 1
    xPositions = range(stepSize * xMin, stepSize * xMax + 1, stepSize)

    yMin = -1 * (shape[0] // 2)
    yMax = yMin + shape[0] - 1
    yPositions = range(stepSize * yMin, stepSize * yMax + 1, stepSize)

    spreadOffsets = list(cross(yPositions, xPositions))

    # Put the (0,0) entry first
    spreadOffsets.remove((0,0))
    spreadOffsets.insert(0, (0,0))

  numSpreadOffsets = len(spreadOffsets)
  print "spreadOffsets:", spreadOffsets

  return centerOffsets, spreadOffsets

Generates centre offsets and spread offsets for block - mode based training regimes - star cross block .


def makeCloneMap ( columnsShape , outputCloningWidth , outputCloningHeight = - 1 ) : if outputCloningHeight < 0 : outputCloningHeight = outputCloningWidth columnsHeight , columnsWidth = columnsShape numDistinctMasters = outputCloningWidth * outputCloningHeight a = numpy . empty ( ( columnsHeight , columnsWidth ) , 'uint32' ) for row in xrange ( columnsHeight ) : for col in xrange ( columnsWidth ) : a [ row , col ] = ( col % outputCloningWidth ) + ( row % outputCloningHeight ) * outputCloningWidth return a , numDistinctMasters

Make a two - dimensional clone map mapping columns to clone master .


def numpyStr(array, format='%f', includeIndices=False, includeZeros=True):
  """ Pretty print a numpy matrix using the given format string for each
  value. Return the string representation

  Parameters:
  ------------------------------------------------------------
  array:    The numpy array to print. This can be either a 1D vector or 2D matrix
  format:   The format string to use for each value
  includeIndices: If true, include [row,col] label for each value
  includeZeros:   Can only be set to False if includeIndices is on.
                  If True, include 0 values in the print-out
                  If False, exclude 0 values from the print-out.

  """

  shape = array.shape
  assert (len(shape) <= 2)
  items = ['[']
  if len(shape) == 1:
    if includeIndices:
      format = '%d:' + format
      if includeZeros:
        rowItems = [format % (c,x) for (c,x) in enumerate(array)]
      else:
        rowItems = [format % (c,x) for (c,x) in enumerate(array) if x != 0]
    else:
      rowItems = [format % (x) for x in array]
    items.extend(rowItems)

  else:
    (rows, cols) = shape
    if includeIndices:
      format = '%d,%d:' + format

    for r in xrange(rows):
      if includeIndices:
        rowItems = [format % (r,c,x) for c,x in enumerate(array[r])]
      else:
        rowItems = [format % (x) for x in array[r]]
      if r > 0:
        items.append('')

      items.append('[')
      items.extend(rowItems)
      if r < rows-1:
        items.append(']\n')
      else:
        items.append(']')

  items.append(']')
  return ' '.join(items)

def
numpyStr
(
array
,
format
=
'%f'
,
includeIndices
=
False
,
includeZeros
=
True
)
:
shape
=
array
.
shape
assert
(
len
(
shape
)
<=
2
)
items
=
[
'['
]
if
len
(
shape
)
==
1
:
if
includeIndices
:
format
=
'%d:'
+
format
if
includeZeros
:
rowItems
=
[
format
%
(
c
,
x
)
for
(
c
,
x
)
in
enumerate
(
array
)
]
else
:
rowItems
=
[
format
%
(
c
,
x
)
for
(
c
,
x
)
in
enumerate
(
array
)
if
x
!=
0
]
else
:
rowItems
=
[
format
%
(
x
)
for
x
in
array
]
items
.
extend
(
rowItems
)
else
:
(
rows
,
cols
)
=
shape
if
includeIndices
:
format
=
'%d,%d:'
+
format
for
r
in
xrange
(
rows
)
:
if
includeIndices
:
rowItems
=
[
format
%
(
r
,
c
,
x
)
for
c
,
x
in
enumerate
(
array
[
r
]
)
]
else
:
rowItems
=
[
format
%
(
x
)
for
x
in
array
[
r
]
]
if
r
>
0
:
items
.
append
(
''
)
items
.
append
(
'['
)
items
.
extend
(
rowItems
)
if
r
<
rows
1
:
items
.
append
(
']\n'
)
else
:
items
.
append
(
']'
)
items
.
append
(
']'
)
return
' '
.
join
(
items
)

Pretty print a numpy matrix using the given format string for each value . Return the string representation


def sample(self, rgen):
    """Generates a random sample from the discrete probability distribution
    and returns its value and the log of the probability of sampling that value.
    """
    rf = rgen.uniform(0, self.sum)
    index = bisect.bisect(self.cdf, rf)
    return self.keys[index], numpy.log(self.pmf[index])

def sample ( self , rgen ) : rf = rgen . uniform ( 0 , self . sum ) index = bisect . bisect ( self . cdf , rf ) return self . keys [ index ] , numpy . log ( self . pmf [ index ] )

Generates a random sample from the discrete probability distribution and returns its value and the log of the probability of sampling that value .


def logProbability(self, distn):
    """Form of distribution must be an array of counts in order of self.keys."""
    x = numpy.asarray(distn)
    n = x.sum()
    return (logFactorial(n) - numpy.sum([logFactorial(k) for k in x]) +
      numpy.sum(x * numpy.log(self.dist.pmf)))

def logProbability ( self , distn ) : x = numpy . asarray ( distn ) n = x . sum ( ) return ( logFactorial ( n ) - numpy . sum ( [ logFactorial ( k ) for k in x ] ) + numpy . sum ( x * numpy . log ( self . dist . pmf ) ) )

Form of distribution must be an array of counts in order of self . keys .


def sample(self, rgen):
    """Generates a random sample from the Poisson probability distribution and
    returns its value and the log of the probability of sampling that value.
    """
    x = rgen.poisson(self.lambdaParameter)
    return x, self.logDensity(x)

def sample ( self , rgen ) : x = rgen . poisson ( self . lambdaParameter ) return x , self . logDensity ( x )

Generates a random sample from the Poisson probability distribution and returns its value and the log of the probability of sampling that value .


def createDataOutLink(network, sensorRegionName, regionName):
  """Link sensor region to other region so that it can pass it data."""
  network.link(sensorRegionName, regionName, "UniformLink", "",
               srcOutput="dataOut", destInput="bottomUpIn")

def createDataOutLink ( network , sensorRegionName , regionName ) : network . link ( sensorRegionName , regionName , "UniformLink" , "" , srcOutput = "dataOut" , destInput = "bottomUpIn" )

Link sensor region to other region so that it can pass it data .


def createSensorToClassifierLinks(network, sensorRegionName,
                                  classifierRegionName):
  """Create required links from a sensor region to a classifier region."""
  network.link(sensorRegionName, classifierRegionName, "UniformLink", "",
               srcOutput="bucketIdxOut", destInput="bucketIdxIn")
  network.link(sensorRegionName, classifierRegionName, "UniformLink", "",
               srcOutput="actValueOut", destInput="actValueIn")
  network.link(sensorRegionName, classifierRegionName, "UniformLink", "",
               srcOutput="categoryOut", destInput="categoryIn")

def createSensorToClassifierLinks ( network , sensorRegionName , classifierRegionName ) : network . link ( sensorRegionName , classifierRegionName , "UniformLink" , "" , srcOutput = "bucketIdxOut" , destInput = "bucketIdxIn" ) network . link ( sensorRegionName , classifierRegionName , "UniformLink" , "" , srcOutput = "actValueOut" , destInput = "actValueIn" ) network . link ( sensorRegionName , classifierRegionName , "UniformLink" , "" , srcOutput = "categoryOut" , destInput = "categoryIn" )

Create required links from a sensor region to a classifier region .


def createNetwork(dataSource):
  """Create and initialize a network."""
  with open(_PARAMS_PATH, "r") as f:
    modelParams = yaml.safe_load(f)["modelParams"]

  # Create a network that will hold the regions.
  network = Network()

  # Add a sensor region.
  network.addRegion("sensor", "py.RecordSensor", '{}')

  # Set the encoder and data source of the sensor region.
  sensorRegion = network.regions["sensor"].getSelf()
  sensorRegion.encoder = createEncoder(modelParams["sensorParams"]["encoders"])
  sensorRegion.dataSource = dataSource

  # Make sure the SP input width matches the sensor region output width.
  modelParams["spParams"]["inputWidth"] = sensorRegion.encoder.getWidth()

  # Add SP and TM regions.
  network.addRegion("SP", "py.SPRegion", json.dumps(modelParams["spParams"]))
  network.addRegion("TM", "py.TMRegion", json.dumps(modelParams["tmParams"]))

  # Add a classifier region.
  clName = "py.%s" % modelParams["clParams"].pop("regionName")
  network.addRegion("classifier", clName, json.dumps(modelParams["clParams"]))

  # Add all links
  createSensorToClassifierLinks(network, "sensor", "classifier")
  createDataOutLink(network, "sensor", "SP")
  createFeedForwardLink(network, "SP", "TM")
  createFeedForwardLink(network, "TM", "classifier")
  # Reset links are optional, since the sensor region does not send resets.
  createResetLink(network, "sensor", "SP")
  createResetLink(network, "sensor", "TM")

  # Make sure all objects are initialized.
  network.initialize()

  return network


def getPredictionResults(network, clRegionName):
  """Get prediction results for all prediction steps."""
  classifierRegion = network.regions[clRegionName]
  actualValues = classifierRegion.getOutputData("actualValues")
  probabilities = classifierRegion.getOutputData("probabilities")
  steps = classifierRegion.getSelf().stepsList
  N = classifierRegion.getSelf().maxCategoryCount
  results = {step: {} for step in steps}
  for i in range(len(steps)):
    # stepProbabilities are probabilities for this prediction step only.
    stepProbabilities = probabilities[i * N:(i + 1) * N - 1]
    mostLikelyCategoryIdx = stepProbabilities.argmax()
    predictedValue = actualValues[mostLikelyCategoryIdx]
    predictionConfidence = stepProbabilities[mostLikelyCategoryIdx]
    results[steps[i]]["predictedValue"] = predictedValue
    results[steps[i]]["predictionConfidence"] = predictionConfidence
  return results

Get prediction results for all prediction steps .


def runHotgym(numRecords):
  """Run the Hot Gym example."""

  # Create a data source for the network.
  dataSource = FileRecordStream(streamID=_INPUT_FILE_PATH)
  numRecords = min(numRecords, dataSource.getDataRowCount())
  network = createNetwork(dataSource)

  # Set predicted field
  network.regions["sensor"].setParameter("predictedField", "consumption")

  # Enable learning for all regions.
  network.regions["SP"].setParameter("learningMode", 1)
  network.regions["TM"].setParameter("learningMode", 1)
  network.regions["classifier"].setParameter("learningMode", 1)

  # Enable inference for all regions.
  network.regions["SP"].setParameter("inferenceMode", 1)
  network.regions["TM"].setParameter("inferenceMode", 1)
  network.regions["classifier"].setParameter("inferenceMode", 1)

  results = []
  N = 1  # Run the network, N iterations at a time.
  for iteration in range(0, numRecords, N):
    network.run(N)

    predictionResults = getPredictionResults(network, "classifier")
    oneStep = predictionResults[1]["predictedValue"]
    oneStepConfidence = predictionResults[1]["predictionConfidence"]
    fiveStep = predictionResults[5]["predictedValue"]
    fiveStepConfidence = predictionResults[5]["predictionConfidence"]

    result = (oneStep, oneStepConfidence * 100,
              fiveStep, fiveStepConfidence * 100)
    print "1-step: {:16} ({:4.4}%)\t 5-step: {:16} ({:4.4}%)".format(*result)
    results.append(result)

  return results


def _loadDummyModelParameters(self, params):
    """ Loads all the parameters for this dummy model. For any paramters
    specified as lists, read the appropriate value for this model using the model
    index """

    for key, value in params.iteritems():
      if type(value) == list:
        index = self.modelIndex % len(params[key])
        self._params[key] = params[key][index]
      else:
        self._params[key] = params[key]

def _loadDummyModelParameters ( self , params ) : for key , value in params . iteritems ( ) : if type ( value ) == list : index = self . modelIndex % len ( params [ key ] ) self . _params [ key ] = params [ key ] [ index ] else : self . _params [ key ] = params [ key ]

Loads all the parameters for this dummy model . For any paramters specified as lists read the appropriate value for this model using the model index


def _getMetrics(self):
    """ Protected function that can be overridden by subclasses. Its main purpose
    is to allow the the OPFDummyModelRunner to override this with deterministic
    values

    Returns: All the metrics being computed for this model
    """
    metric = None
    if self.metrics is not None:
      metric = self.metrics(self._currentRecordIndex+1)
    elif self.metricValue is not None:
      metric = self.metricValue
    else:
      raise RuntimeError('No metrics or metric value specified for dummy model')

    return {self._optimizeKeyPattern:metric}

Protected function that can be overridden by subclasses . Its main purpose is to allow the the OPFDummyModelRunner to override this with deterministic values


def __shouldSysExit(self, iteration):
    """
    Checks to see if the model should exit based on the exitAfter dummy
    parameter
    """

    if self._exitAfter is None \
       or iteration < self._exitAfter:
      return False

    results = self._jobsDAO.modelsGetFieldsForJob(self._jobID, ['params'])

    modelIDs = [e[0] for e in results]
    modelNums = [json.loads(e[1][0])['structuredParams']['__model_num'] for e in results]

    sameModelNumbers = filter(lambda x: x[1] == self.modelIndex,
                              zip(modelIDs, modelNums))

    firstModelID = min(zip(*sameModelNumbers)[0])

    return firstModelID == self._modelID

def __shouldSysExit ( self , iteration ) : if self . _exitAfter is None or iteration < self . _exitAfter : return False results = self . _jobsDAO . modelsGetFieldsForJob ( self . _jobID , [ 'params' ] ) modelIDs = [ e [ 0 ] for e in results ] modelNums = [ json . loads ( e [ 1 ] [ 0 ] ) [ 'structuredParams' ] [ '__model_num' ] for e in results ] sameModelNumbers = filter ( lambda x : x [ 1 ] == self . modelIndex , zip ( modelIDs , modelNums ) ) firstModelID = min ( zip ( * sameModelNumbers ) [ 0 ] ) return firstModelID == self . _modelID


def getDescription(self):
    """Returns a description of the dataset"""

    description = {'name':self.name, 'fields':[f.name for f in self.fields], \
      'numRecords by field':[f.numRecords for f in self.fields]}

    return description

def
getDescription
(
self
)
:
description
=
{
'name'
:
self
.
name
,
'fields'
:
[
f
.
name
for
f
in
self
.
fields
]
,
'numRecords by field'
:
[
f
.
numRecords
for
f
in
self
.
fields
]
}
return
description


def generateRecords(self, records):
    """Generate multiple records. Refer to definition for generateRecord"""

    if self.verbosity>0: print 'Generating', len(records), 'records...'
    for record in records:
      self.generateRecord(record)

def generateRecords ( self , records ) : if self . verbosity > 0 : print 'Generating' , len ( records ) , 'records...' for record in records : self . generateRecord ( record )

Generate multiple records . Refer to definition for generateRecord


def getRecord(self, n=None):
    """Returns the nth record"""

    if n is None:
      assert len(self.fields)>0
      n = self.fields[0].numRecords-1

    assert (all(field.numRecords>n for field in self.fields))

    record = [field.values[n] for field in self.fields]

    return record

def getRecord ( self , n = None ) : if n is None : assert len ( self . fields ) > 0 n = self . fields [ 0 ] . numRecords - 1 assert ( all ( field . numRecords > n for field in self . fields ) ) record = [ field . values [ n ] for field in self . fields ] return record


def getAllRecords(self):
    """Returns all the records"""
    values=[]
    numRecords = self.fields[0].numRecords
    assert (all(field.numRecords==numRecords for field in self.fields))

    for x in range(numRecords):
      values.append(self.getRecord(x))

    return values

def getAllRecords ( self ) : values = [ ] numRecords = self . fields [ 0 ] . numRecords assert ( all ( field . numRecords == numRecords for field in self . fields ) ) for x in range ( numRecords ) : values . append ( self . getRecord ( x ) ) return values


def addValuesToField(self, i, numValues):
    """Add values to the field i."""

    assert(len(self.fields)>i)
    values = [self.addValueToField(i) for n in range(numValues)]
    return values

def addValuesToField ( self , i , numValues ) : assert ( len ( self . fields ) > i ) values = [ self . addValueToField ( i ) for n in range ( numValues ) ] return values


def getSDRforValue(self, i, j):
    """Returns the sdr for jth value at column i"""
    assert len(self.fields)>i
    assert self.fields[i].numRecords>j
    encoding = self.fields[i].encodings[j]

    return encoding

def getSDRforValue ( self , i , j ) : assert len ( self . fields ) > i assert self . fields [ i ] . numRecords > j encoding = self . fields [ i ] . encodings [ j ] return encoding


def getZeroedOutEncoding(self, n):
    """Returns the nth encoding with the predictedField zeroed out"""

    assert all(field.numRecords>n for field in self.fields)

    encoding = np.concatenate([field.encoder.encode(SENTINEL_VALUE_FOR_MISSING_DATA)\
        if field.isPredictedField else field.encodings[n] for field in self.fields])

    return encoding

def getZeroedOutEncoding ( self , n ) : assert all ( field . numRecords > n for field in self . fields ) encoding = np . concatenate ( [ field . encoder . encode ( SENTINEL_VALUE_FOR_MISSING_DATA ) if field . isPredictedField else field . encodings [ n ] for field in self . fields ] ) return encoding


def getTotaln(self):
    """Returns the cumulative n for all the fields in the dataset"""

    n = sum([field.n for field in self.fields])
    return n

def getTotaln ( self ) : n = sum ( [ field . n for field in self . fields ] ) return n


def getTotalw(self):
    """Returns the cumulative w for all the fields in the dataset"""

    w = sum([field.w for field in self.fields])
    return w

def getTotalw ( self ) : w = sum ( [ field . w for field in self . fields ] ) return w


def getEncoding(self, n):
    """Returns the nth encoding"""

    assert (all(field.numEncodings>n for field in self.fields))
    encoding = np.concatenate([field.encodings[n] for field in self.fields])

    return encoding

def getEncoding ( self , n ) : assert ( all ( field . numEncodings > n for field in self . fields ) ) encoding = np . concatenate ( [ field . encodings [ n ] for field in self . fields ] ) return encoding


def getAllEncodings(self):
    """Returns encodings for all the records"""

    numEncodings=self.fields[0].numEncodings
    assert (all(field.numEncodings==numEncodings for field in self.fields))
    encodings = [self.getEncoding(index) for index in range(numEncodings)]

    return encodings

def getAllEncodings ( self ) : numEncodings = self . fields [ 0 ] . numEncodings assert ( all ( field . numEncodings == numEncodings for field in self . fields ) ) encodings = [ self . getEncoding ( index ) for index in range ( numEncodings ) ] return encodings


def saveRecords(self, path='myOutput'):
    """Export all the records into a csv file in numenta format.

    Example header format:
    fieldName1    fieldName2    fieldName3
    date          string        float
    T             S

    Parameters:
    --------------------------------------------------------------------
    path:      Relative path of the file to which the records are to be exported
    """
    numRecords = self.fields[0].numRecords
    assert (all(field.numRecords==numRecords for field in self.fields))

    import csv
    with open(path+'.csv', 'wb') as f:
      writer = csv.writer(f)
      writer.writerow(self.getAllFieldNames())
      writer.writerow(self.getAllDataTypes())
      writer.writerow(self.getAllFlags())
      writer.writerows(self.getAllRecords())
    if self.verbosity>0:
      print '******', numRecords,'records exported in numenta format to file:',\
                path,'******\n'

Export all the records into a csv file in numenta format .


def removeAllRecords(self):
    """Deletes all the values in the dataset"""

    for field in self.fields:
      field.encodings, field.values=[], []
      field.numRecords, field.numEncodings= (0, 0)

def removeAllRecords ( self ) : for field in self . fields : field . encodings , field . values = [ ] , [ ] field . numRecords , field . numEncodings = ( 0 , 0 )


def encodeValue(self, value, toBeAdded=True):
    """Value is encoded as a sdr using the encoding parameters of the Field"""

    encodedValue = np.array(self.encoder.encode(value), dtype=realDType)

    if toBeAdded:
      self.encodings.append(encodedValue)
      self.numEncodings+=1

    return encodedValue

def encodeValue ( self , value , toBeAdded = True ) : encodedValue = np . array ( self . encoder . encode ( value ) , dtype = realDType ) if toBeAdded : self . encodings . append ( encodedValue ) self . numEncodings += 1 return encodedValue


def _setTypes(self, encoderSpec):
    """Set up the dataTypes and initialize encoders"""

    if self.encoderType is None:
      if self.dataType in ['int','float']:
        self.encoderType='adaptiveScalar'
      elif self.dataType=='string':
        self.encoderType='category'
      elif self.dataType in ['date', 'datetime']:
        self.encoderType='date'

    if self.dataType is None:
      if self.encoderType in ['scalar','adaptiveScalar']:
        self.dataType='float'
      elif self.encoderType in ['category', 'enumeration']:
        self.dataType='string'
      elif self.encoderType in ['date', 'datetime']:
        self.dataType='datetime'

def _setTypes ( self , encoderSpec ) : if self . encoderType is None : if self . dataType in [ 'int' , 'float' ] : self . encoderType = 'adaptiveScalar' elif self . dataType == 'string' : self . encoderType = 'category' elif self . dataType in [ 'date' , 'datetime' ] : self . encoderType = 'date' if self . dataType is None : if self . encoderType in [ 'scalar' , 'adaptiveScalar' ] : self . dataType = 'float' elif self . encoderType in [ 'category' , 'enumeration' ] : self . dataType = 'string' elif self . encoderType in [ 'date' , 'datetime' ] : self . dataType = 'datetime'


def _initializeEncoders(self, encoderSpec):
    """ Initialize the encoders"""

    #Initializing scalar encoder
    if self.encoderType in ['adaptiveScalar', 'scalar']:
      if 'minval' in encoderSpec:
        self.minval = encoderSpec.pop('minval')
      else: self.minval=None
      if 'maxval' in encoderSpec:
        self.maxval = encoderSpec.pop('maxval')
      else: self.maxval = None
      self.encoder=adaptive_scalar.AdaptiveScalarEncoder(name='AdaptiveScalarEncoder', \
                                                         w=self.w, n=self.n, minval=self.minval, maxval=self.maxval, periodic=False, forced=True)

    #Initializing category encoder
    elif self.encoderType=='category':
      self.encoder=sdr_category.SDRCategoryEncoder(name='categoryEncoder', \
                                                   w=self.w, n=self.n)

    #Initializing date encoder
    elif self.encoderType in ['date', 'datetime']:
      self.encoder=date.DateEncoder(name='dateEncoder')
    else:
      raise RuntimeError('Error in constructing class object. Either encoder type'
          'or dataType must be specified')


def loadExperiment(path):
  """Loads the experiment description file from the path.

  :param path: (string) The path to a directory containing a description.py file
         or the file itself.
  :returns: (config, control)
  """
  if not os.path.isdir(path):
    path = os.path.dirname(path)
  descriptionPyModule = loadExperimentDescriptionScriptFromDir(path)
  expIface = getExperimentDescriptionInterfaceFromModule(descriptionPyModule)
  return expIface.getModelDescription(), expIface.getModelControl()

Loads the experiment description file from the path .


def loadExperimentDescriptionScriptFromDir(experimentDir):
  """ Loads the experiment description python script from the given experiment
  directory.

  :param experimentDir: (string) experiment directory path

  :returns:        module of the loaded experiment description scripts
  """
  descriptionScriptPath = os.path.join(experimentDir, "description.py")
  module = _loadDescriptionFile(descriptionScriptPath)
  return module

def loadExperimentDescriptionScriptFromDir ( experimentDir ) : descriptionScriptPath = os . path . join ( experimentDir , "description.py" ) module = _loadDescriptionFile ( descriptionScriptPath ) return module

Loads the experiment description python script from the given experiment directory .


def _loadDescriptionFile(descriptionPyPath):
  """Loads a description file and returns it as a module.

  descriptionPyPath: path of description.py file to load
  """
  global g_descriptionImportCount

  if not os.path.isfile(descriptionPyPath):
    raise RuntimeError(("Experiment description file %s does not exist or " + \
                        "is not a file") % (descriptionPyPath,))

  mod = imp.load_source("pf_description%d" % g_descriptionImportCount,
                        descriptionPyPath)
  g_descriptionImportCount += 1

  if not hasattr(mod, "descriptionInterface"):
    raise RuntimeError("Experiment description file %s does not define %s" % \
                       (descriptionPyPath, "descriptionInterface"))

  if not isinstance(mod.descriptionInterface, exp_description_api.DescriptionIface):
    raise RuntimeError(("Experiment description file %s defines %s but it " + \
                        "is not DescriptionIface-based") % \
                            (descriptionPyPath, name))

  return mod

Loads a description file and returns it as a module .


def getModelIDFromParamsHash(self, paramsHash):
    """ Return the modelID of the model with the given paramsHash, or
    None if not found.

    Parameters:
    ---------------------------------------------------------------------
    paramsHash:  paramsHash to look for
    retval:      modelId, or None if not found
    """
    entryIdx = self. _paramsHashToIndexes.get(paramsHash, None)
    if entryIdx is not None:
      return self._allResults[entryIdx]['modelID']
    else:
      return None

def getModelIDFromParamsHash ( self , paramsHash ) : entryIdx = self . _paramsHashToIndexes . get ( paramsHash , None ) if entryIdx is not None : return self . _allResults [ entryIdx ] [ 'modelID' ] else : return None

Return the modelID of the model with the given paramsHash or None if not found .


def bestModelIdAndErrScore(self, swarmId=None, genIdx=None):
    """Return the model ID of the model with the best result so far and
    it's score on the optimize metric. If swarm is None, then it returns
    the global best, otherwise it returns the best for the given swarm
    for all generatons up to and including genIdx.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'
    genIdx:   consider the best in all generations up to and including this
                generation if not None.
    retval:  (modelID, result)
    """
    if swarmId is None:
      return (self._bestModelID, self._bestResult)

    else:
      if swarmId not in self._swarmBestOverall:
        return (None, numpy.inf)

      # Get the best score, considering the appropriate generations
      genScores = self._swarmBestOverall[swarmId]
      bestModelId = None
      bestScore = numpy.inf

      for (i, (modelId, errScore)) in enumerate(genScores):
        if genIdx is not None and i > genIdx:
          break
        if errScore < bestScore:
          bestScore = errScore
          bestModelId = modelId

      return (bestModelId, bestScore)

Return the model ID of the model with the best result so far and it s score on the optimize metric . If swarm is None then it returns the global best otherwise it returns the best for the given swarm for all generatons up to and including genIdx .


def getParticleInfo(self, modelId):
    """Return particle info for a specific modelId.

    Parameters:
    ---------------------------------------------------------------------
    modelId:  which model Id

    retval:  (particleState, modelId, errScore, completed, matured)
    """
    entry = self._allResults[self._modelIDToIdx[modelId]]
    return (entry['modelParams']['particleState'], modelId, entry['errScore'],
            entry['completed'], entry['matured'])

def getParticleInfo ( self , modelId ) : entry = self . _allResults [ self . _modelIDToIdx [ modelId ] ] return ( entry [ 'modelParams' ] [ 'particleState' ] , modelId , entry [ 'errScore' ] , entry [ 'completed' ] , entry [ 'matured' ] )

Return particle info for a specific modelId .


def getParticleInfos(self, swarmId=None, genIdx=None, completed=None,
                       matured=None, lastDescendent=False):
    """Return a list of particleStates for all particles we know about in
    the given swarm, their model Ids, and metric results.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'

    genIdx:  If not None, only return particles at this specific generation
                  index.

    completed:   If not None, only return particles of the given state (either
                completed if 'completed' is True, or running if 'completed'
                is false

    matured:   If not None, only return particles of the given state (either
                matured if 'matured' is True, or not matured if 'matured'
                is false. Note that any model which has completed is also
                considered matured.

    lastDescendent: If True, only return particles that are the last descendent,
                that is, the highest generation index for a given particle Id

    retval:  (particleStates, modelIds, errScores, completed, matured)
              particleStates: list of particleStates
              modelIds: list of modelIds
              errScores: list of errScores, numpy.inf is plugged in
                              if we don't have a result yet
              completed: list of completed booleans
              matured: list of matured booleans
    """
    # The indexes of all the models in this swarm. This list excludes hidden
    #  (orphaned) models.
    if swarmId is not None:
      entryIdxs = self._swarmIdToIndexes.get(swarmId, [])
    else:
      entryIdxs = range(len(self._allResults))
    if len(entryIdxs) == 0:
      return ([], [], [], [], [])

    # Get the particles of interest
    particleStates = []
    modelIds = []
    errScores = []
    completedFlags = []
    maturedFlags = []
    for idx in entryIdxs:
      entry = self._allResults[idx]

      # If this entry is hidden (i.e. it was an orphaned model), it should
      #  not be in this list
      if swarmId is not None:
        assert (not entry['hidden'])

      # Get info on this model
      modelParams = entry['modelParams']
      isCompleted = entry['completed']
      isMatured = entry['matured']
      particleState = modelParams['particleState']
      particleGenIdx = particleState['genIdx']
      particleId = particleState['id']

      if genIdx is not None and particleGenIdx != genIdx:
        continue

      if completed is not None and (completed != isCompleted):
        continue

      if matured is not None and (matured != isMatured):
        continue

      if lastDescendent \
              and (self._particleLatestGenIdx[particleId] != particleGenIdx):
        continue

      # Incorporate into return values
      particleStates.append(particleState)
      modelIds.append(entry['modelID'])
      errScores.append(entry['errScore'])
      completedFlags.append(isCompleted)
      maturedFlags.append(isMatured)

    return (particleStates, modelIds, errScores, completedFlags, maturedFlags)

Return a list of particleStates for all particles we know about in the given swarm their model Ids and metric results .


def getOrphanParticleInfos(self, swarmId, genIdx):
    """Return a list of particleStates for all particles in the given
    swarm generation that have been orphaned.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'

    genIdx:  If not None, only return particles at this specific generation
                  index.

    retval:  (particleStates, modelIds, errScores, completed, matured)
              particleStates: list of particleStates
              modelIds: list of modelIds
              errScores: list of errScores, numpy.inf is plugged in
                              if we don't have a result yet
              completed: list of completed booleans
              matured: list of matured booleans
    """

    entryIdxs = range(len(self._allResults))
    if len(entryIdxs) == 0:
      return ([], [], [], [], [])

    # Get the particles of interest
    particleStates = []
    modelIds = []
    errScores = []
    completedFlags = []
    maturedFlags = []
    for idx in entryIdxs:

      # Get info on this model
      entry = self._allResults[idx]
      if not entry['hidden']:
        continue

      modelParams = entry['modelParams']
      if modelParams['particleState']['swarmId'] != swarmId:
        continue

      isCompleted = entry['completed']
      isMatured = entry['matured']
      particleState = modelParams['particleState']
      particleGenIdx = particleState['genIdx']
      particleId = particleState['id']

      if genIdx is not None and particleGenIdx != genIdx:
        continue

      # Incorporate into return values
      particleStates.append(particleState)
      modelIds.append(entry['modelID'])
      errScores.append(entry['errScore'])
      completedFlags.append(isCompleted)
      maturedFlags.append(isMatured)

    return (particleStates, modelIds, errScores, completedFlags, maturedFlags)

def
getOrphanParticleInfos
(
self
,
swarmId
,
genIdx
)
:
entryIdxs
=
range
(
len
(
self
.
_allResults
)
)
if
len
(
entryIdxs
)
==
0
:
return
(
[
]
,
[
]
,
[
]
,
[
]
,
[
]
)
# Get the particles of interest
particleStates
=
[
]
modelIds
=
[
]
errScores
=
[
]
completedFlags
=
[
]
maturedFlags
=
[
]
for
idx
in
entryIdxs
:
# Get info on this model
entry
=
self
.
_allResults
[
idx
]
if
not
entry
[
'hidden'
]
:
continue
modelParams
=
entry
[
'modelParams'
]
if
modelParams
[
'particleState'
]
[
'swarmId'
]
!=
swarmId
:
continue
isCompleted
=
entry
[
'completed'
]
isMatured
=
entry
[
'matured'
]
particleState
=
modelParams
[
'particleState'
]
particleGenIdx
=
particleState
[
'genIdx'
]
particleId
=
particleState
[
'id'
]
if
genIdx
is
not
and
particleGenIdx
!=
genIdx
:
continue
# Incorporate into return values
particleStates
.
append
(
particleState
)
modelIds
.
append
(
entry
[
'modelID'
]
)
errScores
.
append
(
entry
[
'errScore'
]
)
completedFlags
.
append
(
isCompleted
)
maturedFlags
.
append
(
isMatured
)
return
(
particleStates
,
modelIds
,
errScores
,
completedFlags
,
maturedFlags
)

Return a list of particleStates for all particles in the given swarm generation that have been orphaned .


def firstNonFullGeneration(self, swarmId, minNumParticles):
    """ Return the generation index of the first generation in the given
    swarm that does not have numParticles particles in it, either still in the
    running state or completed. This does not include orphaned particles.

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  A string representation of the sorted list of encoders in this
                 swarm. For example '__address_encoder.__gym_encoder'
    minNumParticles: minium number of partices required for a full
                  generation.

    retval:  generation index, or None if no particles at all.
    """

    if not swarmId in self._swarmNumParticlesPerGeneration:
      return None

    numPsPerGen = self._swarmNumParticlesPerGeneration[swarmId]

    numPsPerGen = numpy.array(numPsPerGen)
    firstNonFull = numpy.where(numPsPerGen < minNumParticles)[0]
    if len(firstNonFull) == 0:
      return len(numPsPerGen)
    else:
      return firstNonFull[0]

def firstNonFullGeneration ( self , swarmId , minNumParticles ) : if not swarmId in self . _swarmNumParticlesPerGeneration : return None numPsPerGen = self . _swarmNumParticlesPerGeneration [ swarmId ] numPsPerGen = numpy . array ( numPsPerGen ) firstNonFull = numpy . where ( numPsPerGen < minNumParticles ) [ 0 ] if len ( firstNonFull ) == 0 : return len ( numPsPerGen ) else : return firstNonFull [ 0 ]

Return the generation index of the first generation in the given swarm that does not have numParticles particles in it either still in the running state or completed . This does not include orphaned particles .


def getResultsPerChoice(self, swarmId, maxGenIdx, varName):
    """ Return a dict of the errors obtained on models that were run with
    each value from a PermuteChoice variable.

    For example, if a PermuteChoice variable has the following choices:
      ['a', 'b', 'c']

    The dict will have 3 elements. The keys are the stringified choiceVars,
    and each value is tuple containing (choiceVar, errors) where choiceVar is
    the original form of the choiceVar (before stringification) and errors is
    the list of errors received from models that used the specific choice:
    retval:
      ['a':('a', [0.1, 0.2, 0.3]), 'b':('b', [0.5, 0.1, 0.6]), 'c':('c', [])]

    Parameters:
    ---------------------------------------------------------------------
    swarmId:  swarm Id of the swarm to retrieve info from
    maxGenIdx: max generation index to consider from other models, ignored
                if None
    varName:  which variable to retrieve

    retval:  list of the errors obtained from each choice.
    """
    results = dict()
    # Get all the completed particles in this swarm
    (allParticles, _, resultErrs, _, _) = self.getParticleInfos(swarmId,
                                              genIdx=None, matured=True)

    for particleState, resultErr in itertools.izip(allParticles, resultErrs):
      # Consider this generation?
      if maxGenIdx is not None:
        if particleState['genIdx'] > maxGenIdx:
          continue

      # Ignore unless this model completed successfully
      if resultErr == numpy.inf:
        continue

      position = Particle.getPositionFromState(particleState)
      varPosition = position[varName]
      varPositionStr = str(varPosition)
      if varPositionStr in results:
        results[varPositionStr][1].append(resultErr)
      else:
        results[varPositionStr] = (varPosition, [resultErr])

    return results

Return a dict of the errors obtained on models that were run with each value from a PermuteChoice variable .


def _getStreamDef(self, modelDescription):
    """
    Generate stream definition based on
    """
    #--------------------------------------------------------------------------
    # Generate the string containing the aggregation settings.
    aggregationPeriod = {
        'days': 0,
        'hours': 0,
        'microseconds': 0,
        'milliseconds': 0,
        'minutes': 0,
        'months': 0,
        'seconds': 0,
        'weeks': 0,
        'years': 0,
    }

    # Honor any overrides provided in the stream definition
    aggFunctionsDict = {}
    if 'aggregation' in modelDescription['streamDef']:
      for key in aggregationPeriod.keys():
        if key in modelDescription['streamDef']['aggregation']:
          aggregationPeriod[key] = modelDescription['streamDef']['aggregation'][key]
      if 'fields' in modelDescription['streamDef']['aggregation']:
        for (fieldName, func) in modelDescription['streamDef']['aggregation']['fields']:
          aggFunctionsDict[fieldName] = str(func)

    # Do we have any aggregation at all?
    hasAggregation = False
    for v in aggregationPeriod.values():
      if v != 0:
        hasAggregation = True
        break

    # Convert the aggFunctionsDict to a list
    aggFunctionList = aggFunctionsDict.items()
    aggregationInfo = dict(aggregationPeriod)
    aggregationInfo['fields'] = aggFunctionList

    streamDef = copy.deepcopy(modelDescription['streamDef'])
    streamDef['aggregation'] = copy.deepcopy(aggregationInfo)
    return streamDef


def _okToExit(self):
    """Test if it's OK to exit this worker. This is only called when we run
    out of prospective new models to evaluate. This method sees if all models
    have matured yet. If not, it will sleep for a bit and return False. This
    will indicate to the hypersearch worker that we should keep running, and
    check again later. This gives this worker a chance to pick up and adopt any
    model which may become orphaned by another worker before it matures.

    If all models have matured, this method will send a STOP message to all
    matured, running models (presummably, there will be just one - the model
    which thinks it's the best) before returning True.
    """
    # Send an update status periodically to the JobTracker so that it doesn't
    # think this worker is dead.
    print >> sys.stderr, "reporter:status:In hypersearchV2: _okToExit"

    # Any immature models still running?
    if not self._jobCancelled:
      (_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(matured=False)
      if len(modelIds) > 0:
        self.logger.info("Ready to end hyperseach, but not all models have " \
                         "matured yet. Sleeping a bit to wait for all models " \
                         "to mature.")
        # Sleep for a bit, no need to check for orphaned models very often
        time.sleep(5.0 * random.random())
        return False

    # All particles have matured, send a STOP signal to any that are still
    # running.
    (_, modelIds, _, _, _) = self._resultsDB.getParticleInfos(completed=False)
    for modelId in modelIds:
      self.logger.info("Stopping model %d because the search has ended" \
                          % (modelId))
      self._cjDAO.modelSetFields(modelId,
                      dict(engStop=ClientJobsDAO.STOP_REASON_STOPPED),
                      ignoreUnchanged = True)

    # Update the HsState to get the accurate field contributions.
    self._hsStatePeriodicUpdate()
    pctFieldContributions, absFieldContributions = \
                                          self._hsState.getFieldContributions()

    # Update the results field with the new field contributions.
    jobResultsStr = self._cjDAO.jobGetFields(self._jobID, ['results'])[0]
    if jobResultsStr is not None:
      jobResults = json.loads(jobResultsStr)
    else:
      jobResults = {}

    # Update the fieldContributions field.
    if pctFieldContributions != jobResults.get('fieldContributions', None):
      jobResults['fieldContributions'] = pctFieldContributions
      jobResults['absoluteFieldContributions'] = absFieldContributions

      isUpdated = self._cjDAO.jobSetFieldIfEqual(self._jobID,
                                                   fieldName='results',
                                                   curValue=jobResultsStr,
                                                   newValue=json.dumps(jobResults))
      if isUpdated:
        self.logger.info('Successfully updated the field contributions:%s',
                                                              pctFieldContributions)
      else:
        self.logger.info('Failed updating the field contributions, ' \
                         'another hypersearch worker must have updated it')

    return True

Test if it s OK to exit this worker . This is only called when we run out of prospective new models to evaluate . This method sees if all models have matured yet . If not it will sleep for a bit and return False . This will indicate to the hypersearch worker that we should keep running and check again later . This gives this worker a chance to pick up and adopt any model which may become orphaned by another worker before it matures .


def recordModelProgress(self, modelID, modelParams, modelParamsHash, results,
                         completed, completionReason, matured, numRecords):
    """Record or update the results for a model. This is called by the
    HSW whenever it gets results info for another model, or updated results
    on a model that is still running.

    The first time this is called for a given modelID, the modelParams will
    contain the params dict for that model and the modelParamsHash will contain
    the hash of the params. Subsequent updates of the same modelID will
    have params and paramsHash values of None (in order to save overhead).

    The Hypersearch object should save these results into it's own working
    memory into some table, which it then uses to determine what kind of
    new models to create next time createModels() is called.

    Parameters:
    ----------------------------------------------------------------------
    modelID:        ID of this model in models table
    modelParams:    params dict for this model, or None if this is just an update
                    of a model that it already previously reported on.

                    See the comments for the createModels() method for a
                    description of this dict.

    modelParamsHash:  hash of the modelParams dict, generated by the worker
                    that put it into the model database.
    results:        tuple containing (allMetrics, optimizeMetric). Each is a
                    dict containing metricName:result pairs. .
                    May be none if we have no results yet.
    completed:      True if the model has completed evaluation, False if it
                      is still running (and these are online results)
    completionReason: One of the ClientJobsDAO.CMPL_REASON_XXX equates
    matured:        True if this model has matured. In most cases, once a
                    model matures, it will complete as well. The only time a
                    model matures and does not complete is if it's currently
                    the best model and we choose to keep it running to generate
                    predictions.
    numRecords:     Number of records that have been processed so far by this
                      model.
    """
    if results is None:
      metricResult = None
    else:
      metricResult = results[1].values()[0]

    # Update our database.
    errScore = self._resultsDB.update(modelID=modelID,
                modelParams=modelParams,modelParamsHash=modelParamsHash,
                metricResult=metricResult, completed=completed,
                completionReason=completionReason, matured=matured,
                numRecords=numRecords)

    # Log message.
    self.logger.debug('Received progress on model %d: completed: %s, '
                      'cmpReason: %s, numRecords: %d, errScore: %s' ,
                      modelID, completed, completionReason, numRecords, errScore)

    # Log best so far.
    (bestModelID, bestResult) = self._resultsDB.bestModelIdAndErrScore()
    self.logger.debug('Best err score seen so far: %s on model %s' % \
                     (bestResult, bestModelID))

Record or update the results for a model . This is called by the HSW whenever it gets results info for another model or updated results on a model that is still running .


def runModel(self, modelID, jobID, modelParams, modelParamsHash,
               jobsDAO, modelCheckpointGUID):
    """Run the given model.

    This runs the model described by 'modelParams'. Periodically, it updates
    the results seen on the model to the model database using the databaseAO
    (database Access Object) methods.

    Parameters:
    -------------------------------------------------------------------------
    modelID:             ID of this model in models table

    jobID:               ID for this hypersearch job in the jobs table

    modelParams:         parameters of this specific model
                         modelParams is a dictionary containing the name/value
                         pairs of each variable we are permuting over. Note that
                         variables within an encoder spec have their name
                         structure as:
                           <encoderName>.<encodrVarName>

    modelParamsHash:     hash of modelParamValues

    jobsDAO              jobs data access object - the interface to the jobs
                          database where model information is stored

    modelCheckpointGUID: A persistent, globally-unique identifier for
                          constructing the model checkpoint key
    """

    # We're going to make an assumption that if we're not using streams, that
    #  we also don't need checkpoints saved. For now, this assumption is OK
    #  (if there are no streams, we're typically running on a single machine
    #  and just save models to files) but we may want to break this out as
    #  a separate controllable parameter in the future
    if not self._createCheckpoints:
      modelCheckpointGUID = None

    # Register this model in our database
    self._resultsDB.update(modelID=modelID,
                           modelParams=modelParams,
                           modelParamsHash=modelParamsHash,
                           metricResult = None,
                           completed = False,
                           completionReason = None,
                           matured = False,
                           numRecords = 0)

    # Get the structured params, which we pass to the base description
    structuredParams = modelParams['structuredParams']

    if self.logger.getEffectiveLevel() <= logging.DEBUG:
      self.logger.debug("Running Model. \nmodelParams: %s, \nmodelID=%s, " % \
                        (pprint.pformat(modelParams, indent=4), modelID))

    # Record time.clock() so that we can report on cpu time
    cpuTimeStart = time.clock()

    # Run the experiment. This will report the results back to the models
    #  database for us as well.
    logLevel = self.logger.getEffectiveLevel()
    try:
      if self._dummyModel is None or self._dummyModel is False:
        (cmpReason, cmpMsg) = runModelGivenBaseAndParams(
                    modelID=modelID,
                    jobID=jobID,
                    baseDescription=self._baseDescription,
                    params=structuredParams,
                    predictedField=self._predictedField,
                    reportKeys=self._reportKeys,
                    optimizeKey=self._optimizeKey,
                    jobsDAO=jobsDAO,
                    modelCheckpointGUID=modelCheckpointGUID,
                    logLevel=logLevel,
                    predictionCacheMaxRecords=self._predictionCacheMaxRecords)
      else:
        dummyParams = dict(self._dummyModel)
        dummyParams['permutationParams'] = structuredParams
        if self._dummyModelParamsFunc is not None:
          permInfo = dict(structuredParams)
          permInfo ['generation'] = modelParams['particleState']['genIdx']
          dummyParams.update(self._dummyModelParamsFunc(permInfo))

        (cmpReason, cmpMsg) = runDummyModel(
                      modelID=modelID,
                      jobID=jobID,
                      params=dummyParams,
                      predictedField=self._predictedField,
                      reportKeys=self._reportKeys,
                      optimizeKey=self._optimizeKey,
                      jobsDAO=jobsDAO,
                      modelCheckpointGUID=modelCheckpointGUID,
                      logLevel=logLevel,
                      predictionCacheMaxRecords=self._predictionCacheMaxRecords)

      # Write out the completion reason and message
      jobsDAO.modelSetCompleted(modelID,
                            completionReason = cmpReason,
                            completionMsg = cmpMsg,
                            cpuTime = time.clock() - cpuTimeStart)

    except InvalidConnectionException, e:
      self.logger.warn("%s", e)


def _engineServicesRunning():
  """ Return true if the engine services are running
  """
  process = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)

  stdout = process.communicate()[0]
  result = process.returncode
  if result != 0:
    raise RuntimeError("Unable to check for running client job manager")

  # See if the CJM is running
  running = False
  for line in stdout.split("\n"):
    if "python" in line and "clientjobmanager.client_job_manager" in line:
      running = True
      break

  return running


def runWithJsonFile(expJsonFilePath, options, outputLabel, permWorkDir):
  """
  Starts a swarm, given a path to a JSON file containing configuration.

  This function is meant to be used with a CLI wrapper that passes command line
  arguments in through the options parameter.

  @param expJsonFilePath {string} Path to a JSON file containing the complete
                                 [swarm description](http://nupic.docs.numenta.org/0.7.0.dev0/guides/swarming/running.html#the-swarm-description).
  @param options {dict} CLI options.
  @param outputLabel {string} Label for output.
  @param permWorkDir {string} Location of working directory.

  @returns {int} Swarm job id.
  """
  if "verbosityCount" in options:
    verbosity = options["verbosityCount"]
    del options["verbosityCount"]
  else:
    verbosity = 1

  _setupInterruptHandling()

  with open(expJsonFilePath, "r") as jsonFile:
    expJsonConfig = json.loads(jsonFile.read())

  outDir = os.path.dirname(expJsonFilePath)
  return runWithConfig(expJsonConfig, options, outDir=outDir,
                       outputLabel=outputLabel, permWorkDir=permWorkDir,
                       verbosity=verbosity)

def runWithJsonFile ( expJsonFilePath , options , outputLabel , permWorkDir ) : if "verbosityCount" in options : verbosity = options [ "verbosityCount" ] del options [ "verbosityCount" ] else : verbosity = 1 _setupInterruptHandling ( ) with open ( expJsonFilePath , "r" ) as jsonFile : expJsonConfig = json . loads ( jsonFile . read ( ) ) outDir = os . path . dirname ( expJsonFilePath ) return runWithConfig ( expJsonConfig , options , outDir = outDir , outputLabel = outputLabel , permWorkDir = permWorkDir , verbosity = verbosity )

Starts a swarm given a path to a JSON file containing configuration .


def runWithPermutationsScript(permutationsFilePath, options,
                                 outputLabel, permWorkDir):
  """
  Starts a swarm, given a path to a permutations.py script.

  This function is meant to be used with a CLI wrapper that passes command line
  arguments in through the options parameter.

  @param permutationsFilePath {string} Path to permutations.py.
  @param options {dict} CLI options.
  @param outputLabel {string} Label for output.
  @param permWorkDir {string} Location of working directory.

  @returns {object} Model parameters.
  """
  global g_currentVerbosityLevel
  if "verbosityCount" in options:
    g_currentVerbosityLevel = options["verbosityCount"]
    del options["verbosityCount"]
  else:
    g_currentVerbosityLevel = 1

  _setupInterruptHandling()

  options["permutationsScriptPath"] = permutationsFilePath
  options["outputLabel"] = outputLabel
  options["outDir"] = permWorkDir
  options["permWorkDir"] = permWorkDir

  # Assume it's a permutations python script
  runOptions = _injectDefaultOptions(options)
  _validateOptions(runOptions)

  return _runAction(runOptions)

Starts a swarm given a path to a permutations . py script .


def _backupFile(filePath):
  """Back up a file

  Parameters:
  ----------------------------------------------------------------------
  retval:         Filepath of the back-up
  """
  assert os.path.exists(filePath)

  stampNum = 0
  (prefix, suffix) = os.path.splitext(filePath)
  while True:
    backupPath = "%s.%d%s" % (prefix, stampNum, suffix)
    stampNum += 1
    if not os.path.exists(backupPath):
      break
  shutil.copyfile(filePath, backupPath)

  return backupPath

def _backupFile ( filePath ) : assert os . path . exists ( filePath ) stampNum = 0 ( prefix , suffix ) = os . path . splitext ( filePath ) while True : backupPath = "%s.%d%s" % ( prefix , stampNum , suffix ) stampNum += 1 if not os . path . exists ( backupPath ) : break shutil . copyfile ( filePath , backupPath ) return backupPath


def _iterModels(modelIDs):
  """Creates an iterator that returns ModelInfo elements for the given modelIDs

  WARNING:      The order of ModelInfo elements returned by the iterator
                may not match the order of the given modelIDs

  Parameters:
  ----------------------------------------------------------------------
  modelIDs:       A sequence of model identifiers (e.g., as returned by
                  _HyperSearchJob.queryModelIDs()).
  retval:         Iterator that returns ModelInfo elements for the given
                  modelIDs (NOTE:possibly in a different order)
  """

  class ModelInfoIterator(object):
    """ModelInfo iterator implementation class
    """

    # Maximum number of ModelInfo elements to load into cache whenever
    # cache empties
    __CACHE_LIMIT = 1000

    debug=False

    def __init__(self, modelIDs):
      """
      Parameters:
      ----------------------------------------------------------------------
      modelIDs:     a sequence of Nupic model identifiers for which this
                    iterator will return _NupicModelInfo instances.
                    NOTE: The returned instances are NOT guaranteed to be in
                    the same order as the IDs in modelIDs sequence.
      retval:       nothing
      """
      # Make our own copy in case caller changes model id list during iteration
      self.__modelIDs = tuple(modelIDs)

      if self.debug:
        _emit(Verbosity.DEBUG,
              "MODELITERATOR: __init__; numModelIDs=%s" % len(self.__modelIDs))

      self.__nextIndex = 0
      self.__modelCache = collections.deque()
      return

    def __iter__(self):
      """Iterator Protocol function

      Parameters:
      ----------------------------------------------------------------------
      retval:         self
      """
      return self

    def next(self):
      """Iterator Protocol function

      Parameters:
      ----------------------------------------------------------------------
      retval:       A _NupicModelInfo instance or raises StopIteration to
                    signal end of iteration.
      """
      return self.__getNext()

    def __getNext(self):
      """Implementation of the next() Iterator Protocol function.

      When the modelInfo cache becomes empty, queries Nupic and fills the cache
      with the next set of NupicModelInfo instances.

      Parameters:
      ----------------------------------------------------------------------
      retval:       A _NupicModelInfo instance or raises StopIteration to
                    signal end of iteration.
      """

      if self.debug:
        _emit(Verbosity.DEBUG,
              "MODELITERATOR: __getNext(); modelCacheLen=%s" % (
                  len(self.__modelCache)))

      if not self.__modelCache:
        self.__fillCache()

      if not self.__modelCache:
        raise StopIteration()

      return self.__modelCache.popleft()

    def __fillCache(self):
      """Queries Nupic and fills an empty modelInfo cache with the next set of
      _NupicModelInfo instances

      Parameters:
      ----------------------------------------------------------------------
      retval:       nothing
      """
      assert (not self.__modelCache)

      # Assemble a list of model IDs to look up
      numModelIDs = len(self.__modelIDs) if self.__modelIDs else 0

      if self.__nextIndex >= numModelIDs:
        return

      idRange = self.__nextIndex + self.__CACHE_LIMIT
      if idRange > numModelIDs:
        idRange = numModelIDs

      lookupIDs = self.__modelIDs[self.__nextIndex:idRange]

      self.__nextIndex += (idRange - self.__nextIndex)

      # Query Nupic for model info of all models in the look-up list
      # NOTE: the order of results may not be the same as lookupIDs
      infoList = _clientJobsDB().modelsInfo(lookupIDs)
      assert len(infoList) == len(lookupIDs), \
            "modelsInfo returned %s elements; expected %s." % \
            (len(infoList), len(lookupIDs))

      # Create _NupicModelInfo instances and add them to cache
      for rawInfo in infoList:
        modelInfo = _NupicModelInfo(rawInfo=rawInfo)
        self.__modelCache.append(modelInfo)

      assert len(self.__modelCache) == len(lookupIDs), \
             "Added %s elements to modelCache; expected %s." % \
             (len(self.__modelCache), len(lookupIDs))

      if self.debug:
        _emit(Verbosity.DEBUG,
              "MODELITERATOR: Leaving __fillCache(); modelCacheLen=%s" % \
                (len(self.__modelCache),))

  return ModelInfoIterator(modelIDs)


def _launchWorkers(self, cmdLine, numWorkers):
    """ Launch worker processes to execute the given command line

    Parameters:
    -----------------------------------------------
    cmdLine: The command line for each worker
    numWorkers: number of workers to launch
    """

    self._workers = []
    for i in range(numWorkers):
      stdout = tempfile.NamedTemporaryFile(delete=False)
      stderr = tempfile.NamedTemporaryFile(delete=False)
      p = subprocess.Popen(cmdLine, bufsize=1, env=os.environ, shell=True,
                           stdin=None, stdout=stdout, stderr=stderr)
      p._stderr_file = stderr
      p._stdout_file = stdout
      self._workers.append(p)

def _launchWorkers ( self , cmdLine , numWorkers ) : self . _workers = [ ] for i in range ( numWorkers ) : stdout = tempfile . NamedTemporaryFile ( delete = False ) stderr = tempfile . NamedTemporaryFile ( delete = False ) p = subprocess . Popen ( cmdLine , bufsize = 1 , env = os . environ , shell = True , stdin = None , stdout = stdout , stderr = stderr ) p . _stderr_file = stderr p . _stdout_file = stdout self . _workers . append ( p )


def __startSearch(self):
    """Starts HyperSearch as a worker or runs it inline for the "dryRun" action

    Parameters:
    ----------------------------------------------------------------------
    retval:         the new _HyperSearchJob instance representing the
                    HyperSearch job
    """
    # This search uses a pre-existing permutations script
    params = _ClientJobUtils.makeSearchJobParamsDict(options=self._options,
                                                     forRunning=True)

    if self._options["action"] == "dryRun":
      args = [sys.argv[0], "--params=%s" % (json.dumps(params))]

      print
      print "=================================================================="
      print "RUNNING PERMUTATIONS INLINE as \"DRY RUN\"..."
      print "=================================================================="
      jobID = hypersearch_worker.main(args)

    else:
      cmdLine = _setUpExports(self._options["exports"])
      # Begin the new search. The {JOBID} string is replaced by the actual
      # jobID returned from jobInsert.
      cmdLine += "$HYPERSEARCH"
      maxWorkers = self._options["maxWorkers"]

      jobID = self.__cjDAO.jobInsert(
        client="GRP",
        cmdLine=cmdLine,
        params=json.dumps(params),
        minimumWorkers=1,
        maximumWorkers=maxWorkers,
        jobType=self.__cjDAO.JOB_TYPE_HS)

      cmdLine = "python -m nupic.swarming.hypersearch_worker" \
                 " --jobID=%d" % (jobID)
      self._launchWorkers(cmdLine, maxWorkers)

    searchJob = _HyperSearchJob(jobID)

    # Save search ID to file (this is used for report generation)
    self.__saveHyperSearchJobID(
      permWorkDir=self._options["permWorkDir"],
      outputLabel=self._options["outputLabel"],
      hyperSearchJob=searchJob)

    if self._options["action"] == "dryRun":
      print "Successfully executed \"dry-run\" hypersearch, jobID=%d" % (jobID)
    else:
      print "Successfully submitted new HyperSearch job, jobID=%d" % (jobID)
      _emit(Verbosity.DEBUG,
            "Each worker executing the command line: %s" % (cmdLine,))

    return searchJob

Starts HyperSearch as a worker or runs it inline for the dryRun action


def loadSavedHyperSearchJob(cls, permWorkDir, outputLabel):
    """Instantiates a _HyperSearchJob instance from info saved in file

    Parameters:
    ----------------------------------------------------------------------
    permWorkDir: Directory path for saved jobID file
    outputLabel: Label string for incorporating into file name for saved jobID
    retval:      _HyperSearchJob instance; raises exception if not found
    """
    jobID = cls.__loadHyperSearchJobID(permWorkDir=permWorkDir,
                                       outputLabel=outputLabel)

    searchJob = _HyperSearchJob(nupicJobID=jobID)
    return searchJob

def loadSavedHyperSearchJob ( cls , permWorkDir , outputLabel ) : jobID = cls . __loadHyperSearchJobID ( permWorkDir = permWorkDir , outputLabel = outputLabel ) searchJob = _HyperSearchJob ( nupicJobID = jobID ) return searchJob


def __saveHyperSearchJobID(cls, permWorkDir, outputLabel, hyperSearchJob):
    """Saves the given _HyperSearchJob instance's jobID to file

    Parameters:
    ----------------------------------------------------------------------
    permWorkDir:   Directory path for saved jobID file
    outputLabel:   Label string for incorporating into file name for saved jobID
    hyperSearchJob: _HyperSearchJob instance
    retval:        nothing
    """
    jobID = hyperSearchJob.getJobID()
    filePath = cls.__getHyperSearchJobIDFilePath(permWorkDir=permWorkDir,
                                                 outputLabel=outputLabel)

    if os.path.exists(filePath):
      _backupFile(filePath)

    d = dict(hyperSearchJobID = jobID)

    with open(filePath, "wb") as jobIdPickleFile:
      pickle.dump(d, jobIdPickleFile)

def __saveHyperSearchJobID ( cls , permWorkDir , outputLabel , hyperSearchJob ) : jobID = hyperSearchJob . getJobID ( ) filePath = cls . __getHyperSearchJobIDFilePath ( permWorkDir = permWorkDir , outputLabel = outputLabel ) if os . path . exists ( filePath ) : _backupFile ( filePath ) d = dict ( hyperSearchJobID = jobID ) with open ( filePath , "wb" ) as jobIdPickleFile : pickle . dump ( d , jobIdPickleFile )

Saves the given _HyperSearchJob instance s jobID to file


def __loadHyperSearchJobID(cls, permWorkDir, outputLabel):
    """Loads a saved jobID from file

    Parameters:
    ----------------------------------------------------------------------
    permWorkDir:  Directory path for saved jobID file
    outputLabel:  Label string for incorporating into file name for saved jobID
    retval:       HyperSearch jobID; raises exception if not found.
    """
    filePath = cls.__getHyperSearchJobIDFilePath(permWorkDir=permWorkDir,
                                                 outputLabel=outputLabel)

    jobID = None
    with open(filePath, "r") as jobIdPickleFile:
      jobInfo = pickle.load(jobIdPickleFile)
      jobID = jobInfo["hyperSearchJobID"]

    return jobID

def __loadHyperSearchJobID ( cls , permWorkDir , outputLabel ) : filePath = cls . __getHyperSearchJobIDFilePath ( permWorkDir = permWorkDir , outputLabel = outputLabel ) jobID = None with open ( filePath , "r" ) as jobIdPickleFile : jobInfo = pickle . load ( jobIdPickleFile ) jobID = jobInfo [ "hyperSearchJobID" ] return jobID


def __getHyperSearchJobIDFilePath(cls, permWorkDir, outputLabel):
    """Returns filepath where to store HyperSearch JobID

    Parameters:
    ----------------------------------------------------------------------
    permWorkDir: Directory path for saved jobID file
    outputLabel: Label string for incorporating into file name for saved jobID
    retval:      Filepath where to store HyperSearch JobID
    """
    # Get the base path and figure out the path of the report file.
    basePath = permWorkDir

    # Form the name of the output csv file that will contain all the results
    filename = "%s_HyperSearchJobID.pkl" % (outputLabel,)
    filepath = os.path.join(basePath, filename)

    return filepath


def emit(self, modelInfo):
    """Emit model info to csv file

    Parameters:
    ----------------------------------------------------------------------
    modelInfo:      _NupicModelInfo instance
    retval:         nothing
    """
    # Open/init csv file, if needed
    if self.__csvFileObj is None:
      # sets up self.__sortedVariableNames and self.__csvFileObj
      self.__openAndInitCSVFile(modelInfo)

    csv = self.__csvFileObj

    # Emit model info row to report.csv
    print >> csv, "%s, " % (self.__searchJobID),
    print >> csv, "%s, " % (modelInfo.getModelID()),
    print >> csv, "%s, " % (modelInfo.statusAsString()),
    if modelInfo.isFinished():
      print >> csv, "%s, " % (modelInfo.getCompletionReason()),
    else:
      print >> csv, "NA, ",
    if not modelInfo.isWaitingToStart():
      print >> csv, "%s, " % (modelInfo.getStartTime()),
    else:
      print >> csv, "NA, ",
    if modelInfo.isFinished():
      dateFormat = "%Y-%m-%d %H:%M:%S"
      startTime = modelInfo.getStartTime()
      endTime = modelInfo.getEndTime()
      print >> csv, "%s, " % endTime,
      st = datetime.strptime(startTime, dateFormat)
      et = datetime.strptime(endTime, dateFormat)
      print >> csv, "%s, " % (str((et - st).seconds)),
    else:
      print >> csv, "NA, ",
      print >> csv, "NA, ",
    print >> csv, "%s, " % str(modelInfo.getModelDescription()),
    print >> csv, "%s, " % str(modelInfo.getNumRecords()),
    paramLabelsDict = modelInfo.getParamLabels()
    for key in self.__sortedVariableNames:
      # Some values are complex structures,.. which need to be represented as
      # strings
      if key in paramLabelsDict:
        print >> csv, "%s, " % (paramLabelsDict[key]),
      else:
        print >> csv, "None, ",
    metrics = modelInfo.getReportMetrics()
    for key in self.__sortedMetricsKeys:
      value = metrics.get(key, "NA")
      value = str(value)
      value = value.replace("\n", " ")
      print >> csv, "%s, " % (value),

    print >> csv


def queryModelIDs(self):
    """Queuries DB for model IDs of all currently instantiated models
    associated with this HyperSearch job.

    See also: _iterModels()

    Parameters:
    ----------------------------------------------------------------------
    retval:         A sequence of Nupic modelIDs
    """
    jobID = self.getJobID()
    modelCounterPairs = _clientJobsDB().modelsGetUpdateCounters(jobID)
    modelIDs = tuple(x[0] for x in modelCounterPairs)

    return modelIDs

def queryModelIDs ( self ) : jobID = self . getJobID ( ) modelCounterPairs = _clientJobsDB ( ) . modelsGetUpdateCounters ( jobID ) modelIDs = tuple ( x [ 0 ] for x in modelCounterPairs ) return modelIDs

Queuries DB for model IDs of all currently instantiated models associated with this HyperSearch job .


def getOptimizationMetricInfo(cls, searchJobParams):
    """Retrives the optimization key name and optimization function.

    Parameters:
    ---------------------------------------------------------
    searchJobParams:
                    Parameter for passing as the searchParams arg to
                    Hypersearch constructor.
    retval:       (optimizationMetricKey, maximize)
                  optimizationMetricKey: which report key to optimize for
                  maximize: True if we should try and maximize the optimizeKey
                    metric. False if we should minimize it.
    """
    if searchJobParams["hsVersion"] == "v2":
      search = HypersearchV2(searchParams=searchJobParams)
    else:
      raise RuntimeError("Unsupported hypersearch version \"%s\"" % \
                         (searchJobParams["hsVersion"]))

    info = search.getOptimizationMetricInfo()
    return info

Retrives the optimization key name and optimization function .


def getAllMetrics(self):
    """Retrives a dictionary of metrics that combines all report and
    optimization metrics

    Parameters:
    ----------------------------------------------------------------------
    retval:         a dictionary of optimization metrics that were collected
                    for the model; an empty dictionary if there aren't any.
    """
    result = self.getReportMetrics()
    result.update(self.getOptimizationMetrics())
    return result

def getAllMetrics ( self ) : result = self . getReportMetrics ( ) result . update ( self . getOptimizationMetrics ( ) ) return result


def getData(self, n):
    """Returns the next n values for the distribution as a list."""

    records = [self.getNext() for x in range(n)]
    return records

def getData ( self , n ) : records = [ self . getNext ( ) for x in range ( n ) ] return records

Returns the next n values for the distribution as a list .


def getTerminationCallbacks(self, terminationFunc):
    """ Returns the periodic checks to see if the model should
    continue running.

    Parameters:
    -----------------------------------------------------------------------
    terminationFunc:  The function that will be called in the model main loop
                      as a wrapper around this function. Must have a parameter
                      called 'index'

    Returns:          A list of PeriodicActivityRequest objects.
    """
    activities = [None] * len(ModelTerminator._MILESTONES)
    for index, (iteration, _) in enumerate(ModelTerminator._MILESTONES):
      cb = functools.partial(terminationFunc, index=index)
      activities[index] = PeriodicActivityRequest(repeating =False,
                                                  period = iteration,
                                                  cb=cb)

def getTerminationCallbacks ( self , terminationFunc ) : activities = [ None ] * len ( ModelTerminator . _MILESTONES ) for index , ( iteration , _ ) in enumerate ( ModelTerminator . _MILESTONES ) : cb = functools . partial ( terminationFunc , index = index ) activities [ index ] = PeriodicActivityRequest ( repeating = False , period = iteration , cb = cb )

Returns the periodic checks to see if the model should continue running .


def getDataRowCount(self):
    """
    Iterates through stream to calculate total records after aggregation.
    This will alter the bookmark state.
    """
    inputRowCountAfterAggregation = 0
    while True:
      record = self.getNextRecord()
      if record is None:
        return inputRowCountAfterAggregation
      inputRowCountAfterAggregation += 1

      if inputRowCountAfterAggregation > 10000:
        raise RuntimeError('No end of datastream found.')

def
getDataRowCount
(
self
)
:
inputRowCountAfterAggregation
=
0
while
True
:
record
=
self
.
getNextRecord
(
)
if
record
is
:
return
inputRowCountAfterAggregation
inputRowCountAfterAggregation
+=
1
if
inputRowCountAfterAggregation
>
10000
:
raise
RuntimeError
(
'No end of datastream found.'
)

Iterates through stream to calculate total records after aggregation . This will alter the bookmark state .


def get(self, number):
    """
    Return a pattern for a number.

    @param number (int) Number of pattern

    @return (set) Indices of on bits
    """
    if not number in self._patterns:
      raise IndexError("Invalid number")

    return self._patterns[number]

def
get
(
self
,
number
)
:
if
not
number
in
self
.
_patterns
:
raise
IndexError
(
"Invalid number"
)
return
self
.
_patterns
[
number
]


def addNoise(self, bits, amount):
    """
    Add noise to pattern.

    @param bits   (set)   Indices of on bits
    @param amount (float) Probability of switching an on bit with a random bit

    @return (set) Indices of on bits in noisy pattern
    """
    newBits = set()

    for bit in bits:
      if self._random.getReal64() < amount:
        newBits.add(self._random.getUInt32(self._n))
      else:
        newBits.add(bit)

    return newBits

def addNoise ( self , bits , amount ) : newBits = set ( ) for bit in bits : if self . _random . getReal64 ( ) < amount : newBits . add ( self . _random . getUInt32 ( self . _n ) ) else : newBits . add ( bit ) return newBits


def numbersForBit(self, bit):
    """
    Return the set of pattern numbers that match a bit.

    @param bit (int) Index of bit

    @return (set) Indices of numbers
    """
    if bit >= self._n:
      raise IndexError("Invalid bit")

    numbers = set()

    for index, pattern in self._patterns.iteritems():
      if bit in pattern:
        numbers.add(index)

    return numbers

def
numbersForBit
(
self
,
bit
)
:
if
bit
>=
self
.
_n
:
raise
IndexError
(
"Invalid bit"
)
numbers
=
set
(
)
for
index
,
pattern
in
self
.
_patterns
.
iteritems
(
)
:
if
bit
in
pattern
:
numbers
.
add
(
index
)
return
numbers

Return the set of pattern numbers that match a bit .


def numberMapForBits(self, bits):
    """
    Return a map from number to matching on bits,
    for all numbers that match a set of bits.

    @param bits (set) Indices of bits

    @return (dict) Mapping from number => on bits.
    """
    numberMap = dict()

    for bit in bits:
      numbers = self.numbersForBit(bit)

      for number in numbers:
        if not number in numberMap:
          numberMap[number] = set()

        numberMap[number].add(bit)

    return numberMap

def numberMapForBits ( self , bits ) : numberMap = dict ( ) for bit in bits : numbers = self . numbersForBit ( bit ) for number in numbers : if not number in numberMap : numberMap [ number ] = set ( ) numberMap [ number ] . add ( bit ) return numberMap

Return a map from number to matching on bits for all numbers that match a set of bits .


def prettyPrintPattern(self, bits, verbosity=1):
    """
    Pretty print a pattern.

    @param bits      (set) Indices of on bits
    @param verbosity (int) Verbosity level

    @return (string) Pretty-printed text
    """
    numberMap = self.numberMapForBits(bits)
    text = ""

    numberList = []
    numberItems = sorted(numberMap.iteritems(),
                         key=lambda (number, bits): len(bits),
                         reverse=True)

    for number, bits in numberItems:

      if verbosity > 2:
        strBits = [str(n) for n in bits]
        numberText = "{0} (bits: {1})".format(number, ",".join(strBits))
      elif verbosity > 1:
        numberText = "{0} ({1} bits)".format(number, len(bits))
      else:
        numberText = str(number)

      numberList.append(numberText)

    text += "[{0}]".format(", ".join(numberList))

    return text

def
prettyPrintPattern
(
self
,
bits
,
verbosity
=
1
)
:
numberMap
=
self
.
numberMapForBits
(
bits
)
text
=
""
numberList
=
[
]
numberItems
=
sorted
(
numberMap
.
iteritems
(
)
,
key
=
lambda
(
number
,
bits
)
:
len
(
bits
)
,
reverse
=
True
)
for
number
,
bits
in
numberItems
:
if
verbosity
>
2
:
strBits
=
[
str
(
n
)
for
n
in
bits
]
numberText
=
"{0} (bits: {1})"
.
format
(
number
,
","
.
join
(
strBits
)
)
elif
verbosity
>
1
:
numberText
=
"{0} ({1} bits)"
.
format
(
number
,
len
(
bits
)
)
else
:
numberText
=
str
(
number
)
numberList
.
append
(
numberText
)
text
+=
"[{0}]"
.
format
(
", "
.
join
(
numberList
)
)
return
text


def _generate(self):
    """
    Generates set of random patterns.
    """
    candidates = np.array(range(self._n), np.uint32)
    for i in xrange(self._num):
      self._random.shuffle(candidates)
      pattern = candidates[0:self._getW()]
      self._patterns[i] = set(pattern)

def _generate ( self ) : candidates = np . array ( range ( self . _n ) , np . uint32 ) for i in xrange ( self . _num ) : self . _random . shuffle ( candidates ) pattern = candidates [ 0 : self . _getW ( ) ] self . _patterns [ i ] = set ( pattern )


def _getW(self):
    """
    Gets a value of `w` for use in generating a pattern.
    """
    w = self._w

    if type(w) is list:
      return w[self._random.getUInt32(len(w))]
    else:
      return w

def _getW ( self ) : w = self . _w if type ( w ) is list : return w [ self . _random . getUInt32 ( len ( w ) ) ] else : return w

Gets a value of w for use in generating a pattern .


def _generate(self):
    """
    Generates set of consecutive patterns.
    """
    n = self._n
    w = self._w

    assert type(w) is int, "List for w not supported"

    for i in xrange(n / w):
      pattern = set(xrange(i * w, (i+1) * w))
      self._patterns[i] = pattern

def
_generate
(
self
)
:
n
=
self
.
_n
w
=
self
.
_w
assert
type
(
w
)
is
int
,
"List for w not supported"
for
i
in
xrange
(
n
/
w
)
:
pattern
=
set
(
xrange
(
i
*
w
,
(
i
+
1
)
*
w
)
)
self
.
_patterns
[
i
]
=
pattern


def inferSingleStep(self, patternNZ, weightMatrix):
    """
    Perform inference for a single step. Given an SDR input and a weight
    matrix, return a predicted distribution.

    :param patternNZ: list of the active indices from the output below
    :param weightMatrix: numpy array of the weight matrix
    :return: numpy array of the predicted class label distribution
    """
    outputActivation = weightMatrix[patternNZ].sum(axis=0)

    # softmax normalization
    outputActivation = outputActivation - numpy.max(outputActivation)
    expOutputActivation = numpy.exp(outputActivation)
    predictDist = expOutputActivation / numpy.sum(expOutputActivation)
    return predictDist

def
inferSingleStep
(
self
,
patternNZ
,
weightMatrix
)
:
outputActivation
=
weightMatrix
[
patternNZ
]
.
sum
(
axis
=
0
)
# softmax normalization
outputActivation
=
outputActivation
numpy
.
max
(
outputActivation
)
expOutputActivation
=
numpy
.
exp
(
outputActivation
)
predictDist
=
expOutputActivation
/
numpy
.
sum
(
expOutputActivation
)
return
predictDist

Perform inference for a single step . Given an SDR input and a weight matrix return a predicted distribution .