def _generateFile(filename, numRecords, categoryList, initProb, firstOrderProb, secondOrderProb, seqLen, numNoise=0, resetsEvery=None):
 print ('Creating %s...' % filename)
 fields = [('reset', 'int', 'R'), ('name', 'string', '')]
 outFile = FileRecordStream(filename, write=True, fields=fields)
 initCumProb = initProb.cumsum()
 firstOrderCumProb = dict()
 for (key, value) in firstOrderProb.iteritems():
  firstOrderCumProb[key] = value.cumsum()
 secondOrderCumProb = dict()
 for (key, value) in secondOrderProb.iteritems():
  secondOrderCumProb[key] = value.cumsum()
 elementsInSeq = []
 numElementsSinceReset = 0
 maxCatIdx = (len(categoryList) - 1)
 for i in xrange(numRecords):
  if (numElementsSinceReset == 0):
   reset = 1
  else:
   reset = 0
  rand = numpy.random.rand()
  if (len(elementsInSeq) == 0):
   catIdx = numpy.searchsorted(initCumProb, rand)
  elif (len(elementsInSeq) == 1):
   catIdx = numpy.searchsorted(firstOrderCumProb[str(elementsInSeq)], rand)
  elif ((len(elementsInSeq) >= 2) and ((seqLen is None) or (len(elementsInSeq) < (seqLen - numNoise)))):
   catIdx = numpy.searchsorted(secondOrderCumProb[str(elementsInSeq[(-2):])], rand)
  else:
   catIdx = numpy.random.randint(len(categoryList))
  catIdx = min(maxCatIdx, catIdx)
  outFile.appendRecord([reset, categoryList[catIdx]])
  elementsInSeq.append(catIdx)
  numElementsSinceReset += 1
  if ((resetsEvery is not None) and (numElementsSinceReset == resetsEvery)):
   numElementsSinceReset = 0
   elementsInSeq = []
  if ((seqLen is not None) and (len(elementsInSeq) == (seqLen + numNoise))):
   elementsInSeq = []
 outFile.close()

def _generateFile ( filename , numRecords , categoryList , initProb , firstOrderProb , secondOrderProb , seqLen , numNoise = 0 , resetsEvery = None ) : print ( 'Creating %s...' % filename ) fields = [ ( 'reset' , 'int' , 'R' ) , ( 'name' , 'string' , '' ) ] outFile = FileRecordStream ( filename , write = True , fields = fields ) initCumProb = initProb . cumsum ( ) firstOrderCumProb = dict ( ) for ( key , value ) in firstOrderProb . iteritems ( ) : firstOrderCumProb [ key ] = value . cumsum ( ) secondOrderCumProb = dict ( ) for ( key , value ) in secondOrderProb . iteritems ( ) : secondOrderCumProb [ key ] = value . cumsum ( ) elementsInSeq = [ ] numElementsSinceReset = 0 maxCatIdx = ( len ( categoryList ) - 1 ) for i in xrange ( numRecords ) : if ( numElementsSinceReset == 0 ) : reset = 1 else : reset = 0 rand = numpy . random . rand ( ) if ( len ( elementsInSeq ) == 0 ) : catIdx = numpy . searchsorted ( initCumProb , rand ) elif ( len ( elementsInSeq ) == 1 ) : catIdx = numpy . searchsorted ( firstOrderCumProb [ str ( elementsInSeq ) ] , rand ) elif ( ( len ( elementsInSeq ) >= 2 ) and ( ( seqLen is None ) or ( len ( elementsInSeq ) < ( seqLen - numNoise ) ) ) ) : catIdx = numpy . searchsorted ( secondOrderCumProb [ str ( elementsInSeq [ ( - 2 ) : ] ) ] , rand ) else : catIdx = numpy . random . randint ( len ( categoryList ) ) catIdx = min ( maxCatIdx , catIdx ) outFile . appendRecord ( [ reset , categoryList [ catIdx ] ] ) elementsInSeq . append ( catIdx ) numElementsSinceReset += 1 if ( ( resetsEvery is not None ) and ( numElementsSinceReset == resetsEvery ) ) : numElementsSinceReset = 0 elementsInSeq = [ ] if ( ( seqLen is not None ) and ( len ( elementsInSeq ) == ( seqLen + numNoise ) ) ) : elementsInSeq = [ ] outFile . close ( )
parameters: filename: name of .