## Mutual information minimization on Toy Gaussians

This is a sanity check involving binary classification of bivariate Gaussians. Data is sampled from bivariate normal distributions as follows:
\begin{align}
x_0 &\sim \mathcal{N}\left(\begin{pmatrix} z \\ 0 \end{pmatrix}, \begin{pmatrix} 1 & 0.5 \\ 0.5 & 1 \end{pmatrix}\right) \\
x_1 &\sim \mathcal{N}\left(\begin{pmatrix} 0 \\ 1.5 \end{pmatrix}, \begin{pmatrix} 1 & 0 \\ 0 & 1 \end{pmatrix}\right)
\end{align}
Where $z \sim \mathcal{N}(0,1)$ simulates uncertainty in the $x$-coordinate of samples drawn from the first distribution. The data is then passed through a shallow neural network, which we refer to as the classification network whose objective is to distinguish between samples drawn from each distribution.

To reduce the dependency of the network on the potentially noisy $x$-variable, we penalize an estimate of the mutual information between the logits of the neural network and the $x$-coordinate of the input data. We optimize a lower bound on the mutual information through a shallow neural network of similar capacity to the classification network, which remains unchanged. The degree to which the output encoding of the classification network should be indepedent of the $x$-coordinate can be controlled through the hyperparameter $\lambda$ in the objective function.


```python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import scipy
from scipy.interpolate import griddata
from sklearn.metrics import roc_curve, auc
plt.style.use('seaborn-darkgrid')
plt.style.use('seaborn-talk')
plt.style.use('seaborn-pastel')
```


```python
import tensorflow as tf
import tensorflow_probability as tfp
import time, os, sys, glob
from lnc import MI
```

    
    WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
    For more information, please see:
      * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
      * https://github.com/tensorflow/addons
    If you depend on functionality not listed there, please file an issue.
    



```python
def MINE(x, y, y_prime, training, batch_size, name='MINE', actv=tf.nn.relu, dimension=2, l2_reg=False):
    """
    Mutual Information Neural Estimator
    (x,y):      Drawn from joint
    y_prime:    Drawn from marginal

    returns
    MI:         Lower bound on mutual information between x,y
    """

    init = tf.contrib.layers.xavier_initializer()
    drop_rate = 0.0
    shape = [32,32,32,32]
    kwargs = {'center': True, 'scale': True, 'training': training, 'fused': True, 'renorm': False}

    lambda_l2 = 0.001
    regularization = tf.contrib.layers.l2_regularizer(scale=lambda_l2) if l2_reg else None
    y_prime = tf.random_shuffle(y)

    def statistic_network(t, name='stat_net', reuse=False):
        with tf.variable_scope(name, initializer=init, reuse=reuse) as scope:

            h0 = tf.layers.dense(t, units=shape[0], activation=actv, kernel_regularizer=regularization) 
            h1 = tf.layers.dense(h0, units=shape[1], activation=actv, kernel_regularizer=regularization) 
            h2 = tf.layers.dense(h1, units=shape[2], activation=actv, kernel_regularizer=regularization) 
            out = tf.layers.dense(h2, units=1, kernel_initializer=init, kernel_regularizer=regularization) 

        return out

    def log_sum_exp_trick(x, batch_size, axis=1):
        # Compute along batch dimension
        x = tf.squeeze(x)
        x_max = tf.reduce_max(x)
        lse = x_max + tf.log(tf.reduce_sum(tf.exp(x-x_max))) - tf.log(batch_size)
        return lse

    x, y, y_prime = tf.expand_dims(tf.squeeze(x), axis=1), tf.expand_dims(tf.squeeze(y), axis=1), tf.expand_dims(tf.squeeze(y_prime), axis=1)
    z = tf.concat([x,y], axis=1)
    z_prime = tf.concat([x,y_prime], axis=1)
    joint_f = statistic_network(z)
    marginal_f = statistic_network(z_prime, reuse=True)

    print('joint shape', joint_f.shape)
    print('marginal shape', marginal_f.shape)
    
    MI_lower_bound = tf.squeeze(tf.reduce_mean(joint_f)) - tf.squeeze(log_sum_exp_trick(marginal_f, 1.*batch_size))

    joint_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=joint_f,
        labels=tf.ones_like(joint_f)))
    marginal_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=marginal_f,
        labels=tf.zeros_like(marginal_f)))

    # Approximate the trained discriminator as the optimal discriminator
    JSD_lower_bound = -(joint_loss + marginal_loss) + tf.log(4.0)

    if args.MI:
        lower_bound = MI_lower_bound
    else:
        lower_bound = JSD_lower_bound

    return lower_bound, joint_f, marginal_f

def dense_network(x, training, name='fully_connected', actv=tf.nn.relu, **kwargs):
    # Toy dense network for binary classification
    init = tf.contrib.layers.xavier_initializer()
    shape = [32, 32, 32, 32, 32]
    kwargs = {'center': True, 'scale': True, 'training': training, 'fused': True, 'renorm': True}

    print('Input X shape', x.get_shape())

    with tf.variable_scope(name, initializer=init, reuse=tf.AUTO_REUSE) as scope:
        h0 = tf.layers.dense(x, units=shape[0], activation=actv)
        # h0 = tf.layers.batch_normalization(h0, **kwargs)
        h1 = tf.layers.dense(h0, units=shape[1], activation=actv)
        # h1 = tf.layers.batch_normalization(h1, **kwargs)
        h2 = tf.layers.dense(h1, units=shape[2], activation=actv)
        # h2 = tf.layers.batch_normalization(h2, **kwargs)
        
    out = tf.layers.dense(h2, units=1, kernel_initializer=init)

    return out

def mutual_information_1D_kraskov(x, y):
    # k-NN based estimate of mutual information
    from lnc import MI
    mi = MI.mi_LNC([x,y],k=5,base=np.exp(1),alpha=0.5)

    return mi

def scope_variables(name):
    with tf.variable_scope(name):
        return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=tf.get_variable_scope().name)
    
def plot_decision_surface(sess, model, title='Decision Surface'):
    data, out = [], []

    for _ in range(42):
        data_, out_ = sess.run([model.data, model.softmax])
        data.append(data_)
        out.append(np.squeeze(out_))

    data = np.vstack(data)
    out = np.hstack(out)
    x_grid, y_grid = np.mgrid[-2:3:100j, -2:4:100j]

    z = griddata(points=data[:,:2], values=out, xi=(x_grid, y_grid), method='cubic')
    plt.contourf(x_grid, y_grid, z, 25, vmin=0., vmax=1., cmap=plt.cm.viridis)
    plt.colorbar(label='Output')
    plt.xlabel('$x$')
    plt.ylabel('$y$')
    plt.title(title)
    plt.scatter([0], [0], c="green", linewidths=0, label=r"$\mu_0$")
    plt.scatter([0], [1.5], c="blue", linewidths=0, label=r"$\mu_1$")
    plt.legend(loc="upper left")
    # plt.savefig(os.path.join('results', 'decision_no_pen.pdf'), format='pdf', dpi=256, bbox_inches='tight')
```


```python
class Model():
    def __init__(self, args):

        tfd = tfp.distributions
        self.global_step = tf.Variable(0, trainable=False)
        self.MI_global_step = tf.Variable(0, trainable=False)

        if args.dims == 2:
            dim = 2
            z = tf.random.normal([args.sample_size])
            z_prime = tf.random.shuffle(z)
            mu_x = tf.stack([z, tf.zeros(args.sample_size)], axis=-1)  # [z,0] broadcasted
            mu_y = [0,1.5]

            cov_x = [[1., 0.5], [0.5, 1.]]
            cov_y = [[1., 0.], [0., 1.]]

            standard_mvn_normal = tfd.MultivariateNormalDiag(loc=np.zeros(dim).astype(np.float32))
            scale_x = tf.cholesky(cov_x)
            scale_y = tf.cholesky(cov_y)

            # u ~ N(0,1), v = scale @ u + loc
            self.sample_x = tf.squeeze(tf.matmul(tf.broadcast_to(scale_x, [args.sample_size, dim, dim]),
                tf.expand_dims(standard_mvn_normal.sample(args.sample_size), axis=-1))) + mu_x
            self.sample_y = tf.squeeze(tf.matmul(tf.broadcast_to(scale_y, [args.sample_size, dim, dim]),
                tf.expand_dims(standard_mvn_normal.sample(args.sample_size), axis=-1))) + mu_y

            self.x_data = tf.concat([self.sample_x, tf.zeros([args.sample_size,1])], axis=1)
            self.y_data = tf.concat([self.sample_y, tf.ones([args.sample_size,1])], axis=1)

            data = tf.random.shuffle(tf.concat([self.x_data, self.y_data], axis=0))
            idx = tf.range(tf.shape(data)[0])
            ridx = tf.random.shuffle(idx)[:args.sample_size]
            self.data = tf.gather(data, ridx)
            self.labels = tf.squeeze(self.data[:,2])

        with tf.variable_scope('classifier', reuse=tf.AUTO_REUSE) as scope:
            self.logits = dense_network(x=self.data[:,:2], training=True)

        with tf.variable_scope('miner', reuse=tf.AUTO_REUSE) as scope:
            self.MI_estimate, joint_f, marginal_f = MINE(x=self.logits, y=self.data[:,0], y_prime=tf.random_shuffle(self.data[:,0]), 
                batch_size=args.sample_size, dimension=dim, training=True, actv=tf.nn.elu)

        self.softmax = tf.nn.sigmoid(self.logits)
        self.pred = tf.squeeze(tf.cast(tf.greater(self.softmax, 0.5), tf.float32))
        self.accuracy = tf.reduce_mean(tf.cast(tf.equal(self.pred, self.labels), tf.float32))

        self.cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits, labels=tf.expand_dims(self.labels,axis=-1))
        self.clf_cost = tf.reduce_mean(self.cross_entropy)
        self.MI_estimate_kNN = tf.py_func(mutual_information_1D_kraskov, inp=[tf.squeeze(self.logits),
            tf.squeeze(self.data[:,0])], Tout=tf.float64)
        
        # Alternate non-saturating updated rule for generator
        # max_E log(D(E,Z)) = min_E H(D(E,Z),1)
        self.NS_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=marginal_f, labels=tf.ones_like(marginal_f)))
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)

        if args.penalty:
            if args.alt_update:
                self.cost = self.clf_cost + args.lambda_MI * tf.nn.relu(self.MI_estimate + self.NS_loss)
            else:
                self.cost = self.clf_cost + args.lambda_MI * tf.nn.relu(self.MI_estimate)
        else:
            self.cost = self.clf_cost
        
        lr = args.lr_E
        theta_f = scope_variables('classifier')
        theta_MINE = scope_variables('miner')
        print('Classifier Parameters', theta_f)
        print('MI-est Parameters', theta_MINE)
        args.optimizer = args.optimizer.lower()
        if args.optimizer == 'adam':
            self.opt = tf.train.AdamOptimizer(lr)
        elif args.optimizer == 'momentum':
            self.opt = tf.train.MomentumOptimizer(lr, momentum=0.9, use_nesterov=True)
        elif args.optimizer == 'rmsprop':
            self.opt = tf.train.RMSPropOptimizer(lr)
        elif args.optimizer == 'sgd':
            self.opt = tf.train.GradientDescentOptimizer(lr)

        with tf.variable_scope('opt', reuse=tf.AUTO_REUSE) as scope:
            self.MI_opt = tf.train.AdamOptimizer(args.lr_T)
            self.grads_clf = self.opt.compute_gradients(self.clf_cost, var_list=theta_f)
            self.grads_mi_clf =  self.opt.compute_gradients(self.MI_estimate, var_list=theta_f)
            self.grads_mi_est = self.MI_opt.compute_gradients(-self.MI_estimate, var_list=theta_MINE)
            self.MINE_opt_op = self.MI_opt.minimize(-self.MI_estimate, global_step=self.MI_global_step,
                    var_list=theta_MINE)
            
            with tf.control_dependencies(update_ops):
                # Ensures that we execute the update_ops before performing the train_step
                self.opt_op = self.opt.minimize(self.cost, global_step=self.global_step, var_list=theta_f)
```


```python
class args(object):
    rho = 0.5
    sample_size = 512
    dims = 2
    lr_E = 1e-4
    lr_T = 1e-3
    optimizer = 'adam'
    name = 'toy-MI'
    lambda_MI = 10.0
    penalty = False
    MI_iters = 4
    MI = False
    alt_update = False
```


```python
def train(args):

    start_time = time.time()
    ckpt = tf.train.get_checkpoint_state('checkpoints')

    # Build graph
    model = Model(args)
    saver = tf.train.Saver()

    sess = tf.InteractiveSession(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False))
    # with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
    sess.run(tf.global_variables_initializer())
    sess.run(tf.local_variables_initializer())
    steps = 5e3
    
    for step in range(int(steps)):
        sess.run(model.opt_op)   
        
        if args.penalty:
            for MI_step in range(args.MI_iters):
                sess.run(model.MINE_opt_op)
                
        if step % 128 == 0:
            if args.dims == 2:
                MI_est, MI_kNN, acc, y, y_hat = sess.run([model.MI_estimate, model.MI_estimate_kNN, model.accuracy,
                    model.labels, model.pred])

                print('MINE: {:.3f} | kNN: {:.3f} | Accuracy: {:.3f} | ({:.2f} s)'.format(MI_est,
                    MI_kNN, acc, time.time()-start_time))

    print("Training Complete. Time elapsed: {:.3f} s".format(time.time()-start_time))
    
    return sess, model
```


```python
sess, model = train(args)
```

    WARNING:tensorflow:From /usr/local/easybuild/software/Tensorflow/1.13.1-intel-2017.u2-GCC-6.2.0-CUDA10-Python-3.6.4-GPU/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
    Instructions for updating:
    Colocations handled automatically by placer.
    Input X shape (512, 2)
    WARNING:tensorflow:From <ipython-input-4-1dba8522a2c9>:72: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
    Instructions for updating:
    Use keras.layers.dense instead.
    joint shape (512, 1)
    marginal shape (512, 1)
    WARNING:tensorflow:From <ipython-input-5-9bd357dc115b>:51: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
    Instructions for updating:
    tf.py_func is deprecated in TF V2. Instead, use
        tf.py_function, which takes a python function which manipulates tf eager
        tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
        an ndarray (just call tensor.numpy()) but having access to eager tensors
        means `tf.py_function`s can use accelerators such as GPUs as well as
        being differentiable using a gradient tape.
        
    Classifier Parameters [<tf.Variable 'classifier/fully_connected/dense/kernel:0' shape=(2, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_1/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_2/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/dense/kernel:0' shape=(32, 1) dtype=float32_ref>, <tf.Variable 'classifier/dense/bias:0' shape=(1,) dtype=float32_ref>]
    MI-est Parameters [<tf.Variable 'miner/stat_net/dense/kernel:0' shape=(2, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_1/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_2/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_3/kernel:0' shape=(32, 1) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_3/bias:0' shape=(1,) dtype=float32_ref>]
    MINE: -0.036 | kNN: 0.645 | Accuracy: 0.385 | (2.29 s)
    MINE: -0.038 | kNN: 0.232 | Accuracy: 0.725 | (2.62 s)
    MINE: -0.037 | kNN: 0.196 | Accuracy: 0.793 | (2.95 s)
    MINE: -0.038 | kNN: 0.142 | Accuracy: 0.783 | (3.29 s)
    MINE: -0.039 | kNN: 0.305 | Accuracy: 0.803 | (3.60 s)
    MINE: -0.035 | kNN: 0.170 | Accuracy: 0.771 | (3.92 s)
    MINE: -0.043 | kNN: 0.238 | Accuracy: 0.793 | (4.25 s)
    MINE: -0.047 | kNN: 0.247 | Accuracy: 0.760 | (4.57 s)
    MINE: -0.044 | kNN: 0.248 | Accuracy: 0.791 | (4.90 s)
    MINE: -0.038 | kNN: 0.184 | Accuracy: 0.783 | (5.22 s)
    MINE: -0.041 | kNN: 0.243 | Accuracy: 0.787 | (5.54 s)
    MINE: -0.041 | kNN: 0.172 | Accuracy: 0.805 | (5.87 s)
    MINE: -0.034 | kNN: 0.159 | Accuracy: 0.795 | (6.19 s)
    MINE: -0.039 | kNN: 0.189 | Accuracy: 0.795 | (6.53 s)
    MINE: -0.040 | kNN: 0.200 | Accuracy: 0.775 | (6.84 s)
    MINE: -0.041 | kNN: 0.184 | Accuracy: 0.787 | (7.17 s)
    MINE: -0.037 | kNN: 0.165 | Accuracy: 0.783 | (7.49 s)
    MINE: -0.037 | kNN: 0.099 | Accuracy: 0.770 | (7.81 s)
    MINE: -0.041 | kNN: 0.238 | Accuracy: 0.803 | (8.13 s)
    MINE: -0.038 | kNN: 0.137 | Accuracy: 0.771 | (8.44 s)
    MINE: -0.051 | kNN: 0.262 | Accuracy: 0.789 | (8.76 s)
    MINE: -0.041 | kNN: 0.190 | Accuracy: 0.812 | (9.09 s)
    MINE: -0.039 | kNN: 0.184 | Accuracy: 0.779 | (9.42 s)
    MINE: -0.039 | kNN: 0.171 | Accuracy: 0.781 | (9.74 s)
    MINE: -0.039 | kNN: 0.149 | Accuracy: 0.775 | (10.06 s)
    MINE: -0.038 | kNN: 0.196 | Accuracy: 0.799 | (10.38 s)
    MINE: -0.042 | kNN: 0.188 | Accuracy: 0.791 | (10.71 s)
    MINE: -0.042 | kNN: 0.196 | Accuracy: 0.805 | (11.03 s)
    MINE: -0.049 | kNN: 0.229 | Accuracy: 0.787 | (11.36 s)
    MINE: -0.041 | kNN: 0.137 | Accuracy: 0.834 | (11.67 s)
    MINE: -0.040 | kNN: 0.136 | Accuracy: 0.805 | (11.99 s)
    MINE: -0.043 | kNN: 0.217 | Accuracy: 0.793 | (12.32 s)
    MINE: -0.038 | kNN: 0.165 | Accuracy: 0.768 | (12.65 s)
    MINE: -0.043 | kNN: 0.109 | Accuracy: 0.766 | (12.97 s)
    MINE: -0.042 | kNN: 0.195 | Accuracy: 0.777 | (13.29 s)
    MINE: -0.037 | kNN: 0.157 | Accuracy: 0.805 | (13.64 s)
    MINE: -0.034 | kNN: 0.193 | Accuracy: 0.760 | (13.96 s)
    MINE: -0.043 | kNN: 0.277 | Accuracy: 0.777 | (14.28 s)
    MINE: -0.043 | kNN: 0.146 | Accuracy: 0.799 | (14.61 s)
    MINE: -0.042 | kNN: 0.219 | Accuracy: 0.814 | (14.92 s)
    Training Complete. Time elapsed: 14.936 s



```python
plot_decision_surface(sess, model, title='Unpenalized Decision Surface')
```

### Switch on MI penalty


```python
sess.close()
args.penalty = True
args.penalty = 1.0
sess_p, model_p = train(args)
```

    Input X shape (512, 2)
    joint shape (512, 1)
    marginal shape (512, 1)
    Classifier Parameters [<tf.Variable 'classifier/fully_connected/dense/kernel:0' shape=(2, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_1/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_2/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/dense/kernel:0' shape=(32, 1) dtype=float32_ref>, <tf.Variable 'classifier/dense/bias:0' shape=(1,) dtype=float32_ref>]
    MI-est Parameters [<tf.Variable 'miner/stat_net/dense/kernel:0' shape=(2, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_1/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_2/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_3/kernel:0' shape=(32, 1) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_3/bias:0' shape=(1,) dtype=float32_ref>]
    MINE: -0.003 | kNN: 0.109 | Accuracy: 0.273 | (1.26 s)
    MINE: 0.003 | kNN: 0.225 | Accuracy: 0.664 | (2.94 s)
    MINE: 0.006 | kNN: 0.117 | Accuracy: 0.621 | (4.52 s)
    MINE: 0.002 | kNN: 0.050 | Accuracy: 0.672 | (6.10 s)
    MINE: -0.001 | kNN: 0.033 | Accuracy: 0.711 | (7.69 s)
    MINE: -0.003 | kNN: 0.024 | Accuracy: 0.658 | (9.28 s)
    MINE: 0.002 | kNN: -0.007 | Accuracy: 0.705 | (10.87 s)
    MINE: -0.002 | kNN: 0.091 | Accuracy: 0.688 | (12.45 s)
    MINE: -0.000 | kNN: 0.024 | Accuracy: 0.719 | (14.03 s)
    MINE: 0.002 | kNN: 0.079 | Accuracy: 0.693 | (15.63 s)
    MINE: 0.001 | kNN: 0.042 | Accuracy: 0.746 | (17.20 s)
    MINE: 0.001 | kNN: 0.073 | Accuracy: 0.697 | (18.78 s)
    MINE: 0.003 | kNN: 0.034 | Accuracy: 0.746 | (20.36 s)
    MINE: 0.001 | kNN: 0.022 | Accuracy: 0.750 | (21.94 s)
    MINE: 0.002 | kNN: -0.018 | Accuracy: 0.723 | (23.51 s)
    MINE: 0.000 | kNN: 0.002 | Accuracy: 0.725 | (25.08 s)
    MINE: 0.001 | kNN: 0.023 | Accuracy: 0.771 | (26.65 s)
    MINE: 0.003 | kNN: 0.025 | Accuracy: 0.750 | (28.23 s)
    MINE: -0.000 | kNN: 0.035 | Accuracy: 0.766 | (29.80 s)
    MINE: 0.000 | kNN: 0.041 | Accuracy: 0.756 | (31.39 s)
    MINE: -0.003 | kNN: 0.031 | Accuracy: 0.760 | (32.96 s)
    MINE: 0.003 | kNN: 0.075 | Accuracy: 0.760 | (34.54 s)
    MINE: 0.004 | kNN: 0.024 | Accuracy: 0.785 | (36.11 s)
    MINE: 0.004 | kNN: 0.058 | Accuracy: 0.783 | (37.69 s)
    MINE: 0.003 | kNN: 0.045 | Accuracy: 0.799 | (39.26 s)
    MINE: -0.001 | kNN: 0.044 | Accuracy: 0.768 | (40.80 s)
    MINE: 0.004 | kNN: 0.015 | Accuracy: 0.773 | (42.34 s)
    MINE: 0.002 | kNN: 0.044 | Accuracy: 0.773 | (43.87 s)
    MINE: -0.001 | kNN: 0.023 | Accuracy: 0.762 | (45.41 s)
    MINE: 0.003 | kNN: 0.025 | Accuracy: 0.777 | (46.97 s)
    MINE: 0.002 | kNN: -0.022 | Accuracy: 0.764 | (48.51 s)
    MINE: -0.001 | kNN: 0.057 | Accuracy: 0.801 | (50.06 s)
    MINE: 0.000 | kNN: 0.052 | Accuracy: 0.744 | (51.60 s)
    MINE: 0.000 | kNN: 0.003 | Accuracy: 0.732 | (53.15 s)
    MINE: -0.004 | kNN: 0.045 | Accuracy: 0.756 | (54.69 s)
    MINE: 0.001 | kNN: 0.021 | Accuracy: 0.773 | (56.23 s)
    MINE: 0.003 | kNN: 0.068 | Accuracy: 0.797 | (57.78 s)
    MINE: 0.000 | kNN: 0.017 | Accuracy: 0.754 | (59.32 s)
    MINE: 0.002 | kNN: 0.020 | Accuracy: 0.795 | (60.87 s)
    MINE: -0.000 | kNN: -0.009 | Accuracy: 0.797 | (62.42 s)
    Training Complete. Time elapsed: 62.500 s



```python
plot_decision_surface(sess_p, model_p, title='Penalized Decision Surface')
```

### Turn $\lambda$ up a bit


```python
sess_p.close()
args.penalty = True
args.lambda_MI = 10.0
sess_p, model_p = train(args)
```

    Input X shape (512, 2)
    joint shape (512, 1)
    marginal shape (512, 1)
    Classifier Parameters [<tf.Variable 'classifier/fully_connected/dense/kernel:0' shape=(2, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_1/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_2/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'classifier/fully_connected/dense_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'classifier/dense/kernel:0' shape=(32, 1) dtype=float32_ref>, <tf.Variable 'classifier/dense/bias:0' shape=(1,) dtype=float32_ref>]
    MI-est Parameters [<tf.Variable 'miner/stat_net/dense/kernel:0' shape=(2, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_1/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_1/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_2/kernel:0' shape=(32, 32) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_2/bias:0' shape=(32,) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_3/kernel:0' shape=(32, 1) dtype=float32_ref>, <tf.Variable 'miner/stat_net/dense_3/bias:0' shape=(1,) dtype=float32_ref>]
    MINE: 0.008 | kNN: 0.586 | Accuracy: 0.525 | (1.21 s)
    MINE: 0.054 | kNN: 0.183 | Accuracy: 0.496 | (2.77 s)
    MINE: 0.001 | kNN: 0.084 | Accuracy: 0.602 | (4.31 s)
    MINE: -0.000 | kNN: 0.104 | Accuracy: 0.619 | (6.04 s)
    MINE: 0.003 | kNN: 0.072 | Accuracy: 0.623 | (7.59 s)
    MINE: 0.005 | kNN: 0.064 | Accuracy: 0.633 | (9.13 s)
    MINE: -0.004 | kNN: -0.009 | Accuracy: 0.590 | (10.69 s)
    MINE: 0.006 | kNN: 0.107 | Accuracy: 0.641 | (12.23 s)
    MINE: 0.001 | kNN: 0.025 | Accuracy: 0.662 | (13.77 s)
    MINE: 0.002 | kNN: 0.079 | Accuracy: 0.648 | (15.32 s)
    MINE: 0.002 | kNN: 0.006 | Accuracy: 0.660 | (16.88 s)
    MINE: 0.002 | kNN: 0.022 | Accuracy: 0.676 | (18.43 s)
    MINE: -0.001 | kNN: 0.078 | Accuracy: 0.670 | (19.98 s)
    MINE: 0.003 | kNN: 0.024 | Accuracy: 0.697 | (21.53 s)
    MINE: 0.002 | kNN: 0.046 | Accuracy: 0.691 | (23.08 s)
    MINE: 0.001 | kNN: 0.021 | Accuracy: 0.672 | (24.62 s)
    MINE: 0.003 | kNN: 0.088 | Accuracy: 0.652 | (26.16 s)
    MINE: 0.002 | kNN: 0.078 | Accuracy: 0.682 | (27.71 s)
    MINE: -0.001 | kNN: 0.001 | Accuracy: 0.658 | (29.26 s)
    MINE: 0.004 | kNN: 0.057 | Accuracy: 0.680 | (30.81 s)
    MINE: 0.002 | kNN: 0.027 | Accuracy: 0.658 | (32.36 s)
    MINE: -0.003 | kNN: 0.092 | Accuracy: 0.688 | (33.91 s)
    MINE: 0.004 | kNN: 0.065 | Accuracy: 0.689 | (35.46 s)
    MINE: 0.001 | kNN: 0.014 | Accuracy: 0.711 | (37.00 s)
    MINE: 0.003 | kNN: 0.057 | Accuracy: 0.693 | (38.54 s)
    MINE: -0.001 | kNN: 0.055 | Accuracy: 0.719 | (40.07 s)
    MINE: -0.000 | kNN: -0.002 | Accuracy: 0.754 | (41.62 s)
    MINE: 0.004 | kNN: 0.040 | Accuracy: 0.727 | (43.17 s)
    MINE: 0.005 | kNN: 0.070 | Accuracy: 0.703 | (44.70 s)
    MINE: -0.001 | kNN: 0.007 | Accuracy: 0.693 | (46.24 s)
    MINE: 0.002 | kNN: 0.036 | Accuracy: 0.715 | (47.79 s)
    MINE: 0.001 | kNN: 0.081 | Accuracy: 0.717 | (49.33 s)
    MINE: -0.001 | kNN: -0.005 | Accuracy: 0.727 | (50.87 s)
    MINE: 0.001 | kNN: 0.039 | Accuracy: 0.707 | (52.42 s)
    MINE: 0.005 | kNN: 0.033 | Accuracy: 0.723 | (53.96 s)
    MINE: 0.000 | kNN: 0.010 | Accuracy: 0.734 | (55.50 s)
    MINE: -0.002 | kNN: 0.037 | Accuracy: 0.762 | (57.05 s)
    MINE: 0.001 | kNN: 0.040 | Accuracy: 0.740 | (58.59 s)
    MINE: 0.002 | kNN: 0.073 | Accuracy: 0.721 | (60.14 s)
    MINE: 0.000 | kNN: 0.015 | Accuracy: 0.754 | (61.69 s)
    Training Complete. Time elapsed: 61.771 s



```python
plot_decision_surface(sess_p, model_p, title='Penalized Decision Surface, $\lambda=10$')
```

dbe1722a885208f986275fa0fbb793f89318207b
ipynb
Jupyter Notebook
notebooks/toy_MI.ipynb
Justin-Tan/invariant_reps
a6a647b03193e02af9f6a289a9443266564de15d
Apache-2.0
2020-02-14T03:18:47.000Z
2021-02-05T05:33:14.000Z
notebooks/toy_MI.ipynb
Justin-Tan/invariant_reps
a6a647b03193e02af9f6a289a9443266564de15d
Apache-2.0
notebooks/toy_MI.ipynb
Justin-Tan/invariant_reps
a6a647b03193e02af9f6a289a9443266564de15d
Apache-2.0
Qwen/Qwen-72B
1. YES
2. YES