def parseCommandLine(argv,extraArgConsumer=None,extraArgSpec=[],extraArgUsage=[]):
    def usage():
    def status(msg): logging.info('%s time %.3f sec mem %.3f Gb' % (msg,time.time()-startTime,util.memusage()))
def isUncachefromSrc(s): return s.find("|")>=0
def getCacheSrcPair(s): return s.split("|")
def makeCacheSrcPair(s1,s2): return "%s|%s" % (s1,s2)
def parseDatasetSpec(spec,db):
def parseDBSpec(spec):
def parseProgSpec(spec,db,proppr=False): class Dataset(object):
    def __init__(self,xDict,yDict):
    def isSinglePredicate(self):
    def extractMode(self,mode):
    def modesToLearn(self):
    def hasMode(self,mode):
    def getX(self,mode):
    def getY(self,mode):
    def size(self):
    def shuffle(self):
    def minibatchIterator(self,batchSize=100,shuffleFirst=True):
    def pprint(self):
    def serialize(self,dir):
    def deserialize(dir):
    def uncacheExamples(dsetFile,db,exampleFile,proppr=True):
    def uncacheMatrix(dsetFile,db,functorToLearn,functorInDB):
    def loadMatrix(db,functorToLearn,functorInDB):
    def _parseLine(line,proppr=True):
    def loadProPPRExamples(db,fileName):
    def loadExamples(db,fileName,proppr=False):
        def getId(typeName,symbol):
    def saveProPPRExamples(self,fileName,db,append=False,mode=None): THING = '__THING__' # name of default type
NULL_ENTITY_NAME = '__NULL__'  #name of null entity marker
OOV_ENTITY_NAME = '__OOV__'  #name of out-of-vocabulary marker entity
class AbstractSchema(object):
class UntypedSchema(AbstractSchema):
class TypedSchema(AbstractSchema):
class SymbolTable(object):
  def checkTyping(self):
  def isTypeless(self):
  def defaultType(self):
  def getTypes(self):
  def insertType(self,typeName):
  def getDomain(self,functor,arity):
  def getRange(self,functor,arity):
  def getArgType(self,functor,arity,i):
  def declarePredicateTypes(self,functor,types):
  def serialize(self,direc):
  def deserialize(direc):
  def getMaxId(self,typeName):
  def hasId(self,typeName,sym):
  def getId(self,typeName,sym):
  def getSymbol(self,typeName,symbolId):
  def _safeSymbTab(self):
  def _checkAndInsertSymbols(self,typeName,symbolFile):
  def __init__(self):
  def checkTyping(self,predicateList):
  def isTypeless(self):
  def empty(self):
  def defaultType(self):
  def getTypes(self):
  def insertType(self,typeName):
  def getDomain(self,functor,arity):
  def getRange(self,functor,arity):
  def getArgType(self,functor,arity,i):
  def declarePredicateTypes(self,functor,types):
  def serialize(self,direc):
  def serializeTo(self,fpLike):
  def deserializeFrom(fileLike):
  def getMaxId(self,typeName):
  def hasId(self,typeName,sym):
  def getId(self,typeName,sym):
  def getSymbol(self,typeName,symbolId):
  def __init__(self):
  def __str__(self):
  def checkTyping(self,predicateList,strict=False):
  def isTypeless(self):
  def insertType(self,typeName):
  def defaultType(self):
  def getTypes(self):
  def getDomain(self,functor,arity):
  def getRange(self,functor,arity):
  def getArgType(self,functor,arity,i):
  def declarePredicateTypes(self,functor,types):
  def _declarePredicateArgType(self,functor,arity,i,typeName):
  def serialize(self,direc):
  def serializeTo(self,fp):
  def deserializeFrom(fileLike):
  def getMaxId(self,typeName):
  def hasId(self,typeName,sym):
  def getId(self,typeName,sym):
  def getSymbol(self,typeName,symbolId):
  def __init__(self,initSymbols=[]):
  def insert(self,symbol):
  def getSymbolList(self):
  def getSymbol(self,id):
  def hasId(self,symbol):
  def getId(self,symbol):
  def getMaxId(self): def asMode(spec):
    def __init__(self,goal):
    def args(self):
    def arg(self,i):
    def getArity(self):
    def getFunctor(self):
    def __str__(self):
    def __repr__(self):
    def __hash__(self):
    def __eq__(self,other):
    def __init__(self,goal,strict=True):
    def isInput(self,i):
    def isOutput(self,i):
    def isConst(self,i):
    def __str__(self):
    def __init__(self,goal):
    def getType(self,i):
    def typeSet(self):
class AbstractDeclaration(object):
class ModeDeclaration(AbstractDeclaration):
class TypeDeclaration(AbstractDeclaration): def fulltype(o):
    def __init__(self,configDict):
    def run(self):
    def _run(self,
             prog=None, trainData=None, testData=None, targetMode=None,
             savedTestPredictions=None, savedTestExamples=None, savedTrainExamples=None, savedModel=None,
             learner=None):
            def doit():
    def predictionAsProPPRSolutions(fileName,theoryPred,db,X,P,append=False,start=0):
    def timeAction(msg, act):
    def printStats(modelMsg,testSet,goldData,predictedData):
class Expt(object):
        TP0 = Expt.timeAction(
            'running untrained theory on train data',
            lambda:learner.datasetPredict(trainData))
          UP0 = Expt.timeAction(
              'running untrained theory on test data',
              lambda:learner.datasetPredict(testData))
        TP1 = Expt.timeAction(
            'running trained theory on train data',
            lambda:learner.datasetPredict(trainData))
          UP1 = Expt.timeAction(
              'running trained theory on test data',
              lambda:learner.datasetPredict(testData)) class Function(object):
class OpSeqFunction(Function):
class NullFunction(Function):
class LogFunction(Function):
class SumFunction(Function):
class SoftmaxFunction(Function):
    def __init__(self):
    def eval(self,db,values,pad):
    def backprop(self,delta,gradAccum,pad):
    def _checkDuplications(self):
    def install(self,nextId=1):
    def copy(self):
    def pprint(self,depth=0):
    def pprintSummary(self):
    def pprintComment(self):
    def children(self):
    def __init__(self,opInputs,opOutput,ops,rule=None,inputTypes=None,outputType=None):
    def __repr__(self):
    def pprintSummary(self):
    def pprintComment(self):
    def _doEval(self,db,values,pad):
    def _doBackprop(self,delta,gradAccum,pad):
    def children(self):
    def copy(self):
    def __init__(self,lhsMode):
    def __repr__(self):
    def pprintSummary(self):
    def _doEval(self,db,values,pad):
    def _doBackprop(self,delta,gradAccum,pad):
    def children(self):
    def copy(self):
    def __init__(self,fun):
    def __repr__(self):
    def pprintSummary(self):
    def _doEval(self,db,values,pad):
    def _doBackprop(self,delta,gradAccum,pad):
    def children(self):
    def copy(self):
    def __init__(self,funs):
    def __repr__(self):
    def pprintSummary(self):
    def _doEval(self,db,values,pad):
    def _doBackprop(self,delta,gradAccum,pad):
    def children(self):
    def copy(self):
    def __init__(self,fun):
    def __repr__(self):
    def pprintSummary(self):
    def _doEval(self,db,values,pad):
    def _doBackprop(self,delta,pad):
    def children(self):
    def copy(self):   DEBUGGER_AVAILABLE = True
  DEBUGGER_AVAILABLE = False
      X = self.db.onehot(sym)
class Interp(object):
    def __init__(self,prog,trainData=None,testData=None):
    def help(self):
    def set(self,depth=None,echo=None,normalize=None,maxTraceMsg=-1,trace=None):
    def list(self,str=None):
    def _listRules(self,functor,arity):
    def _listAllRules(self):
    def _listFacts(self,functor,arity):
    def _listAllFacts(self):
    def _listFunction(self,modeSpec):
    def eval(self,modeSpec,sym,inputType=None,outputType=None):
    def debug(self,modeSpec,sym):
    def helpConfig(self):
    def debugDset(self,modeSpec,test=False): class GradAccumulator(object):
class Tracer(object):
class EpochTracer(Tracer):
class Learner(object):
class OnePredFixedRateGDLearner(Learner):
class FixedRateGDLearner(Learner):
class FixedRateSGDLearner(FixedRateGDLearner):
class Regularizer(object):
class NullRegularizer(object):
class L2Regularizer(Regularizer):
    def __init__(self):
    def keys(self):
    def items(self):
    def __getitem__(self,paramName):
    def __setitem__(self,paramName,gradient):
    def accum(self,paramName,deltaGradient):
    def fitParameterShapes(self):
    def mapData(self,mapFun):
    def addedTo(self,other):
    def counter():
    def mergeCounters(counters):
    def accumToCounter(counter,otherCounter):
    def _mergeCountersWithInit(counters,init):
    def silent(learner,gradAccum,Y,P,**kw):
    def cheap(learner,gradAccum,Y,P,**kw):
    def default(learner,gradAccum,Y,P,**kw):
    def recordDefaults(learner,gradAccum,Y,P,**kw):
    def defaultPlusAcc(learner,gradAccum,Y,P,**kw):
    def _announce(gradAccum,keyValuePairList):
    def _record(gradAccum,keyValuePairList):
    def loss(learner,Y,P,kw):
    def accuracy(learner,Y,P,kw):
    def timing(learner,kw):
    def identification(learner,kw):
    def silent(learner,ctr,**kw):
    def cheap(learner,ctr,**kw):
    def default(learner,ctr,**kw):
    def __init__(self,prog,regularizer,tracer,epochTracer):
    def predict(self,mode,X,pad=None):
    def datasetPredict(self,dset,copyXs=True):
    def datasetAccuracy(goldDset,predictedDset):
    def datasetCrossEntropy(goldDset,predictedDset,perExample=True):
    def accuracy(Y,P):
        def allZerosButArgmax(d):
    def crossEntropy(Y,P,perExample=False):
    def crossEntropyGrad(self,mode,X,Y,tracerArgs={},pad=None):
    def meanUpdate(self,functor,arity,delta,n,totalN=0):
    def applyUpdate(self,paramGrads,rate):
    def __init__(self,prog,epochs=10,rate=0.1,regularizer=None,tracer=None,epochTracer=None):
    def train(self,mode,X,Y):
    def __init__(self,prog,epochs=10,rate=0.1,regularizer=None,tracer=None,epochTracer=None):
    def train(self,dset):
    def __init__(self,prog,epochs=10,rate=0.1,regularizer=None,tracer=None,miniBatchSize=100):
    def train(self,dset):
    def regularizeParams(self,prog,n):
    def regularizationCost(self,prog):
    def regularizeParams(self,prog,n):
    def regularizationCost(self,prog):
    def __init__(self,regularizationConstant=0.01):
    def regularizeParams(self,prog,n):
    def regularizationCost(self,prog):
            X = dset.getX(mode)
            Y = goldDset.getY(mode)
            P = predictedDset.getY(mode)
            Y = goldDset.getY(mode)
            P = predictedDset.getY(mode)
        P = self.predict(mode,X,pad) NULL_ENTITY_NAME = dbschema.NULL_ENTITY_NAME
THING = dbschema.THING
OOV_ENTITY_NAME = dbschema.OOV_ENTITY_NAME
TRAINABLE_DECLARATION_FUNCTOR = 'trainable'
class MatrixDB(object):
  def __init__(self,initSchema=None):
  def checkTyping(self):
  def isTypeless(self):
  def _fillDefault(self,typeName):
  def dim(self,typeName=None):
  def onehot(self,s,typeName=None,outOfVocabularySymbolsAllowed=False):
  def zeros(self,numRows=1,typeName=None):
  def ones(self,typeName=None):
  def nullMatrix(self,numRows=1,typeName=None,numCols=0):
  def transposeNeeded(mode,transpose=False):
  def matrix(self,mode,transpose=False):
  def vector(self,mode):
  def matrixPreimage(self,mode):
  def matrixPreimageMat(self,mode):
  def matrixPreimageOnesType(self,mode):
  def matrixPreimageOnes(self,mode):
  def isParameter(self,mode):
  def markAsParam(self,functor,arity):
  def markAsParameter(self,functor,arity):
  def clearParameterMarkings(self):
  def getParameter(self,functor,arity):
  def parameterIsInitialized(self,functor,arity):
  def setParameter(self,functor,arity,replacement):
  def asSymbol(self,symbolId,typeName=None):
  def asSymbolId(self,symbol,typeName=None):
  def rowAsSymbolDict(self,row,typeName=None):
  def arrayAsSymbolDict(self,arr,typeName=None):
  def matrixAsSymbolDict(self,m,typeName=None):
  def matrixAsPredicateFacts(self,functor,arity,m):
  def inDB(self,functor,arity):
  def summary(self,functor,arity):
  def listing(self):
  def numMatrices(self):
  def size(self):
  def parameterSize(self):
  def createPartner(self):
  def serialize(self,direc):
  def serializeDataTo(self,fileLike,filter=None):
  def importSerializedDataFrom(self,fileLike):
  def deserializeDataFrom(fileLike):
  def _saveMatDictWithScipy(fileLike,d):
  def _restoreMatDictWithScipy(fileLike):
  def deserialize(direc):
  def uncache(dbFile,factFile,initSchema=None):
  def addLines(self,lines):
  def loadFile(filenames,initSchema=None):
  def startBuffers(self):
  def bufferFile(self,filename):
  def flushBuffers(self):
  def _flushBuffer(self,functor,arity):
  def _bufferTriplet(self,functor,arity,a1,a2,w,filename,k):
  def _bufferLine(self,line,filename,k):
    def _atof(s): NONETYPE=type(None) 
        D = SS.csr_matrix((mat.data,mat.indices-loIndex,mat.indptr),shape=newShape,dtype='float32').todense()
    D1 = SS.csr_matrix((m1.data,m1.indices-loIndex,m1.indptr),shape=newShape1,dtype='float32').todense()
    D2 = SS.csr_matrix((m2.data,m2.indices-loIndex,m2.indptr),shape=newShape2,dtype='float32').todense()
def summary(mat):
def pprintSummary(mat):
def checkCSR(mat,context='unknown'):
def checkNoNANs(mat,context='unknown'):
def maxValue(mat):
def densify(mat,maxExpandFactor=-1,maxExpandIntercept=-1):
def denseSize(m,loIndex,hiIndex):
def sparseSize(m,loIndex,hiIndex):
def codensify(m1,m2,maxExpandFactor=-1,maxExpandIntercept=-1):
def undensify(denseMat, info):
def mean(mat):
def rowsum(mat):
def mapData(dataFun,mat):
def stack(mats):
def numRows(m):
def numCols(m):
def nzCols(m,i):
def repeat(row,n):
def alterMatrixRows(mat,alterationFun):
def softmax(db,mat):
        def softMaxAlteration(data,lo,hi,unused):
def denseSoftmax(m):
def broadcastAndComponentwiseMultiply(m1,m2):
def multiplyByBroadcastRowVec(m,v):
def broadcastAndWeightByRowSum(m1,m2):
def shuffleRows(m,shuffledRowNums=None):
def selectRows(m,lo,hi): def isProcessedConstant(a):
def isProcessedVariable(a):
def isVariableAtom(a):
    def __init__(self,functor,args):
    def _setArgs(self,args):
    def __str__(self):
    def __repr__(self):
    def __init__(self,lhs,rhs,features=None,findall=None):
    def variabilize(self):
            def convertArgs(args):
            def convertGoal(g):
    def __str__(self):
    def asString(self,syntax=None):
    def __init__(self,syntax=None):
    def _key(self,g):
    def add(self,r):
    def size(self):
    def rulesFor(self,g):
    def mapRules(self,mapfun):
    def listing(self):
    def __iter__(self):
    def equals(self,other):
  def __init__(self,syntax=None):
  def setSyntax(self,syntax):
  def _convertGoal(self,ptree):
  def _convertRule(self,ptree):
  def parseGoal(self,s):
  def parseGoalList(self,s):
  def parseRule(self,s):
  def parseQuery(self,s):
  def parseFile(self,filename,rules=None):
  def parseStream(self,fileLike,rules=None):
class Goal(object):
class Rule(object):
class RuleCollection(object):
class Parser(object): def _initWorker(learnerClass,*args):
def _doBackpropTask(task):
def _doAcceptNewParams(paramDict):
def _doPredict(miniBatch):
    def __init__(self,prog,epochs=10,rate=0.1,regularizer=None,tracer=None,
                 miniBatchSize=100,parallel=10,epochTracer=None):
    def datasetPredict(self,dset,copyXs=True):
    def miniBatchToTask(batch,i,k,startTime):
    def totalNumExamples(self,miniBatches):
    def processGradients(self,bpOutputs,totalN):
    def broadcastParameters(self):
    def train(self,dset):
    def __init__(self,prog,**kw):
    def train(self,dset):
class ParallelFixedRateGDLearner(learn.FixedRateSGDLearner):
class ParallelAdaGradLearner(ParallelFixedRateGDLearner): class Program(object):
class ProPPRProgram(Program):
class Plugins(object):
    def __init__(self, db=None, rules=parser.RuleCollection(), plugins=None, calledFromProPPRProgram=False):
        def checkRule(r):
    def clearFunctionCache(self):
    def findPredDef(self,mode):
    def compile(self,mode,depth=0):
    def getPredictFunction(self,mode):
    def getParamList(self):
    def getFunction(self,mode):
    def evalSymbols(self,mode,symbols,typeName=None):
    def eval(self,mode,inputs):
    def evalGradSymbols(self,mode,symbols):
    def evalGrad(self,mode,inputs):
    def setAllWeights(self):
    def setFeatureWeights(self,epsilon=1.0):
    def setRuleWeights(self,weights=None,epsilon=1.0):
    def _loadRules(fileNames):
    def loadRules(fileNames,db,plugins=None):
    def serialize(self,direc):
    def deserialize(direc):
    def serializeRulesTo(self,fileLike):
    def deserializeRulesFrom(fileLike):
    def __init__(self, db=None, rules=parser.RuleCollection(), weights=None, plugins=None):
    def setRuleWeights(self,weights=None,epsilon=1.0,ruleIdPred=None):
    def getRuleWeights(self):
    def setFeatureWeights(self,epsilon=1.0):
        def possibleModes(rule):
    def _setFeatureWeightsForTypelessDB(self,epsilon=1.0):
            def typeOfWeights(mode):
    def setFeatureWeight(self,paramName,arity,weight):
    def _moveFeaturesToRHS(self,rule0):
    def loadRules(fileNames,db,plugins=None):
  def __init__(self):
  def isempty(self):
  def define(self,mode,outputFun,outputTypeFun=None):
  def isDefined(self,mode=None,functor=None,arity=None):
  def definition(self,mode):
  def outputType(self,mode,inputTypes): def memusage():
def linesIn(fileLike):
import unittest
import logging
import logging.config
import collections
import sys
import math
import os
import os.path
import shutil
import tempfile
import scipy
from tensorlog import comline
from tensorlog import dataset
from tensorlog import dbschema
from tensorlog import declare
from tensorlog import expt
from tensorlog import funs
from tensorlog import interp
from tensorlog import learn
from tensorlog import matrixdb
from tensorlog import mutil
from tensorlog import parser
from tensorlog import plearn
from tensorlog import program
from tensorlog import util
# (C) William W. Cohen and Carnegie Mellon University, 2016

# can call a single test with, e.g.,
# python -m unittest testtensorlog.TestSmallProofs.testIf

# NOTE: google style guide for python is 2-space indents, so I'm
# trying to switch to that - W




TEST_DATA_DIR = os.path.join(os.path.dirname(__file__),"test-data/")

def softmax_normalize(expected_result_dict):
  """Compute the softmax of the values in a dictionary, i.e.,
  exponentiate and normalize."""
  for k in expected_result_dict:
    expected_result_dict[k] = math.exp(expected_result_dict[k])
  norm = sum(expected_result_dict.values())
  for c in expected_result_dict:
    expected_result_dict[c] /= norm

def rules_from_strings(rule_strings):
  """Convert a list of strings to a RuleCollection"""
  for r in rule_strings:
    print('>',r)

    rules = parser.RuleCollection()