def mutualInformation(sp, activeColumnsCurrentEpoch, column_1, column_2):
  """
  Computes the mutual information of the binary variables that represent 
  the activation probabilities of two columns. The mutual information I(X,Y) 
  of two random variables is given by
  \[
       I (X,Y)  = \sum_{x,y} p(x,y) log( p(x,y) / ( p(x) p(y) ) ).
  \]
  (https://en.wikipedia.org/wiki/Mutual_information)
  """
  i, j        = column_1, column_2
  batchSize   = activeColumnsCurrentEpoch.shape[0]

  # Activity Counts
  ci, cj, cij = 0., 0., dict([((0,0),0.), ((1,0),0.), ((0,1),0.), ((1,1),0.)])
  for t in range(batchSize):
    ai = activeColumnsCurrentEpoch[t, i]
    aj = activeColumnsCurrentEpoch[t, j]
    cij[(ai, aj)] += 1.
    ci += ai
    cj += aj

  # Mutual information calculation 
  Iij = 0
  for a,b in [(0,0), (1,0), (0,1), (1,1)]:
    # Compute probabilities
    pij = cij[(a,b)]/batchSize
    pi  = ci/batchSize if a == 1 else 1. - ci/batchSize
    pj  = cj/batchSize if b == 1 else 1. - cj/batchSize
    # Add current term of mutual information 
    Iij += pij * np.log2(pij/(pi*pj)) if pij > 0 else 0

  return Iij
def mutualInformation ( sp , activeColumnsCurrentEpoch , column_1 , column_2 ) : i , j = column_1 , column_2 batchSize = activeColumnsCurrentEpoch . shape [ 0 ] ci , cj , cij = 0. , 0. , dict ( [ ( ( 0 , 0 ) , 0. ) , ( ( 1 , 0 ) , 0. ) , ( ( 0 , 1 ) , 0. ) , ( ( 1 , 1 ) , 0. ) ] ) for t in range ( batchSize ) : ai = activeColumnsCurrentEpoch [ t , i ] aj = activeColumnsCurrentEpoch [ t , j ] cij [ ( ai , aj ) ] += 1. ci += ai cj += aj Iij = 0 for a , b in [ ( 0 , 0 ) , ( 1 , 0 ) , ( 0 , 1 ) , ( 1 , 1 ) ] : pij = cij [ ( a , b ) ] / batchSize pi = ci / batchSize if a == 1 else 1. - ci / batchSize pj = cj / batchSize if b == 1 else 1. - cj / batchSize Iij += pij * np . log2 ( pij / ( pi * pj ) ) if pij > 0 else 0 return Iij
Computes the mutual information of the binary variables that represent 
  the activation probabilities of two columns. The mutual information I(X,Y) 
  of two random variables is given by
  \[
       I (X,Y)  = \sum_{x,y} p(x,y) log( p(x,y) / ( p(x) p(y) ) ).
  \]
  (https://en.wikipedia.org/wiki/Mutual_information)