<reponame>davidstutz/disentangling-robustness-generalization
import os
import sys

sys.path.insert(1, os.path.dirname(os.path.realpath(__file__)) + '/../')
from common import utils
from common.log import log
from common import paths

import numpy
import argparse
import sklearn.decomposition
import sklearn.neighbors
import terminaltables


if utils.display():
    from common import plot


class TestAttackClassifier:
    """
    Test a trained classifier.
    """

    def __init__(self, args=None):
        """
        Initialize.

        :param args: optional arguments if not to use sys.argv
        :type args: [str]
        """

        self.args = None
        """ Arguments of program. """

        parser = self.get_parser()
        if args is not None:
            self.args = parser.parse_args(args)
        else:
            self.args = parser.parse_args()

        self.test_images = None
        """ (numpy.ndarray) Images to test on. """

        self.perturbation_images = None
        """ (numpy.ndarray) Images corresponding to perturbations. """

        self.train_images = None
        """ (numpy.ndarray) Images to train on. """

        self.test_theta = None
        """ (numpy.ndarray) Transformation parameters. """

        self.train_theta = None
        """ (numpy.ndarra) Transformation parameters, i.e. latent codes. """

        self.perturbations = None
        """ (numpy.ndarray) Perturbations per test image. """

        self.test_codes = None
        """ (numpy.ndarray) Test codes."""

        self.perturbation_codes = None
        """ (numpy.ndarray) Codes corresponding to perturbations. """

        self.accuracy = None
        """ (numpy.ndarray) Success indicator of test samples. """

        self.success = None
        """ (numpy.ndarray) Success indicator for perturbations."""

        self.pca = None
        """ (sklearn.decomposition.IncrementalPCA) PCA to make nearest neighbor more efficient. """

        self.neighbors = None
        """ (sklearn.neighbors.NearestNeighbors) Nearest neighbor model. """

        self.norms = [1, 2, float('inf')]
        """ ([float]) Norms to evaluate. """

        self.results = []
        """ (dict) Dictionary containing all statistics. """

        for n in range(len(self.norms)):
            self.results.append(dict())

        log('-- ' + self.__class__.__name__)
        for key in vars(self.args):
            log('[Testing] %s=%s' % (key, str(getattr(self.args, key))))

    def get_parser(self):
        """
        Get parser.

        :return: parser
        :rtype: argparse.ArgumentParser
        """

        parser = argparse.ArgumentParser(description='Test attacks on classifier.')
        parser.add_argument('-test_images_file', default=paths.test_images_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-train_images_file', default=paths.train_images_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-test_theta_file', default=paths.test_theta_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-train_theta_file', default=paths.train_theta_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-test_codes_file', default=paths.test_codes_file(), help='HDF5 file containing labels.', type=str)
        parser.add_argument('-label_index', default=2, help='Column index in label file.', type=int)
        parser.add_argument('-accuracy_file', default=paths.results_file('classifier/accuracy'), help='Correctly classified test samples of classifier.', type=str)
        parser.add_argument('-perturbations_file', default=paths.results_file('classifier/perturbations'), help='HDF5 file containing perturbations.', type=str)
        parser.add_argument('-success_file', default=paths.results_file('classifier/success'), help='HDF5 file indicating attack success.', type=str)
        parser.add_argument('-results_file', default='', help='Path to pickled results file.', type=str)
        parser.add_argument('-plot_directory', default=paths.experiment_dir('classifier'), help='Path to PNG plot file for success rate.', type=str)
        parser.add_argument('-plot_manifolds', default=False, action='store_true', help='Whether to plot manifolds.')
        parser.add_argument('-latent', default=False, action='store_true', help='Latent statistics.')

        return parser

    def load_data(self):
        """
        Load data and model.
        """

        self.test_images = utils.read_hdf5(self.args.test_images_file)
        self.test_images = self.test_images.reshape((self.test_images.shape[0], -1))
        log('[Testing] read %s' % self.args.test_images_file)

        self.train_images = utils.read_hdf5(self.args.train_images_file)
        self.train_images = self.train_images.reshape((self.train_images.shape[0], -1))
        log('[Testing] read %s' % self.args.train_images_file)

        self.train_theta = utils.read_hdf5(self.args.train_theta_file)
        log('[Testing] read %s' % self.args.train_theta_file)

        self.test_theta = utils.read_hdf5(self.args.test_theta_file)
        log('[Testing] read %s' % self.args.test_theta_file)

        self.test_codes = utils.read_hdf5(self.args.test_codes_file)
        self.test_codes = self.test_codes[:, self.args.label_index]
        log('[Testing] read %s' % self.args.test_codes_file)

        self.perturbations = utils.read_hdf5(self.args.perturbations_file)
        if len(self.perturbations.shape) > 3:
            self.perturbations = self.perturbations.reshape((self.perturbations.shape[0], self.perturbations.shape[1], -1))
        self.perturbation_images = self.test_images[:self.perturbations.shape[1]]
        self.perturbation_codes = self.test_codes[:self.perturbations.shape[1]]
        log('[Testing] read %s' % self.args.perturbations_file)
        assert not numpy.any(self.perturbations != self.perturbations), 'NaN in perturbations'

        self.success = utils.read_hdf5(self.args.success_file)
        log('[Testing] read %s' % self.args.success_file)

        self.accuracy = utils.read_hdf5(self.args.accuracy_file)
        self.accuracy = self.accuracy[:self.perturbations.shape[1]]
        log('[Testing] read %s' % self.args.accuracy_file)

    def compute_nearest_neighbors(self, images):
        """
        Compute distances in image and latent space.

        :param images: images to get nearest neighbors for
        :type images: numpy.ndarray
        :param norm: norm to use
        :type norm: float
        """

        fit = 100000
        if self.pca is None:
            self.pca = sklearn.decomposition.IncrementalPCA(n_components=20)
            self.pca.fit(self.train_images[:fit])
            log('[Testing] fitted PCA')
        if self.neighbors is None:
            data = self.pca.transform(self.train_images)
            self.neighbors = sklearn.neighbors.NearestNeighbors(n_neighbors=10, algorithm='kd_tree')
            self.neighbors.fit(data[:fit])
            log('[Testing] fitted nearest neighbor')

        data = self.pca.transform(images)
        _, indices = self.neighbors.kneighbors(data)
        return indices

    def compute_statistics(self):
        """
        Compute statistics based on distances.
        """

        N_class = numpy.max(self.test_codes) + 1
        num_attempts = self.perturbations.shape[0]

        perturbations = numpy.swapaxes(self.perturbations, 0, 1)
        perturbations = perturbations.reshape((perturbations.shape[0]*perturbations.shape[1], perturbations.shape[2]))
        success = numpy.swapaxes(self.success, 0, 1)
        success = success.reshape((success.shape[0]*success.shape[1]))

        accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0)
        # Raw success is the base for all statistics, as we need to consider only these
        # attacks that are successful and where the classifier originally was correct.
        raw_overall_success = numpy.logical_and(success >= 0, accuracy)
        log('[Testing] %d valid attacks' % numpy.sum(raw_overall_success))

        # For off-manifold attacks this should not happen, but save is save.
        if not numpy.any(raw_overall_success):
            for n in range(len(self.norms)):
                for type in ['raw_success', 'raw_iteration', 'raw_average', 'raw_latent']:
                    self.results[n][type] = 0
                for type in ['raw_class_success', 'raw_class_average', 'raw_class_latent']:
                    self.results[n][type] = numpy.zeros((N_class))
            if self.args.results_file:
                utils.write_pickle(self.args.results_file, self.results)
                log('[Testing] wrote %s' % self.args.results_file)
            log('[Testing] no successful attacks found, no plots')
            return

        perturbation_images = numpy.repeat(self.perturbation_images, num_attempts, axis=0)
        perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)

        #
        # We compute some simple statistics:
        # - raw success rate: fraction of successful attack without considering epsilon
        # - corrected success rate: fraction of successful attacks within epsilon-ball
        # - raw average perturbation: average distance to original samples (for successful attacks)
        # - corrected average perturbation: average distance to original samples for perturbations
        #   within epsilon-ball (for successful attacks).
        # These statistics can also be computed per class.
        # And these statistics are computed with respect to three norms.

        if self.args.plot_directory and utils.display():
            iterations = success[raw_overall_success]
            x = numpy.arange(numpy.max(iterations) + 1)
            y = numpy.bincount(iterations)
            plot_file = os.path.join(self.args.plot_directory, 'iterations')
            plot.bar(plot_file, x, y, title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count')
            log('[Testing] wrote %s' % plot_file)

        for n in range(len(self.norms)):
            norm = self.norms[n]
            delta = numpy.linalg.norm(perturbation_images - perturbations, norm, axis=1)

            if self.args.plot_directory and utils.display():
                plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm)
                plot.histogram(plot_file, delta[raw_overall_success], 50, title='Distribution of $L_{%g}$ Distances of Successful Attacks' % norm,
                              xlabel='Distance', ylabel='Count')
                log('[Testing] wrote %s' % plot_file)

            #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0]
            #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0)
            #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy)
            #log('[Testing] attacked model accuracy: %g' % debug_accuracy)
            #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
            #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)

            N_accuracy = numpy.sum(accuracy)
            self.results[n]['raw_success'] = numpy.sum(raw_overall_success) / N_accuracy
            self.results[n]['raw_iteration'] = numpy.average(success[raw_overall_success])
            self.results[n]['raw_average'] = numpy.average(delta[raw_overall_success]) if numpy.any(raw_overall_success) else 0
            self.results[n]['raw_latent'] = 0

            raw_class_success = numpy.zeros((N_class, perturbation_images.shape[0]), bool)
            self.results[n]['raw_class_success'] = numpy.zeros((N_class))
            self.results[n]['raw_class_average'] = numpy.zeros((N_class))
            self.results[n]['raw_class_latent'] = numpy.zeros((N_class))

            for c in range(N_class):
                N_samples = numpy.sum(numpy.logical_and(accuracy, perturbation_codes == c))
                if N_samples <= 0:
                    continue;

                raw_class_success[c] = numpy.logical_and(raw_overall_success, perturbation_codes == c)
                self.results[n]['raw_class_success'][c] = numpy.sum(raw_class_success[c]) / N_samples
                if numpy.any(raw_class_success[c]):
                    self.results[n]['raw_class_average'][c] = numpy.average(delta[raw_class_success[c].astype(bool)])

        if self.args.results_file:
            utils.write_pickle(self.args.results_file, self.results)
            log('[Testing] wrote %s' % self.args.results_file)

    def compute_latent_statistics(self):
        """
        Compute latent statistics.
        """

        N_class = numpy.max(self.test_codes) + 1
        num_attempts = self.perturbations.shape[0]

        perturbations = numpy.swapaxes(self.perturbations, 0, 1)
        perturbations = perturbations.reshape((perturbations.shape[0] * perturbations.shape[1], perturbations.shape[2]))
        success = numpy.swapaxes(self.success, 0, 1)
        success = success.reshape((success.shape[0] * success.shape[1]))

        accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0)
        # Raw success is the base for all statistics, as we need to consider only these
        # attacks that are successful and where the classifier originally was correct.
        raw_overall_success = numpy.logical_and(success >= 0, accuracy)

        # For off-manifold attacks this should not happen, but save is save.
        if not numpy.any(raw_overall_success):
            for n in range(len(self.norms)):
                for type in ['raw_success', 'raw_iteration', 'raw_average', 'raw_latent']:
                    self.results[n][type] = 0
                for type in ['raw_class_success', 'raw_class_average', 'raw_class_latent']:
                    self.results[n][type] = numpy.zeros((N_class))
            if self.args.results_file:
                utils.write_pickle(self.args.results_file, self.results)
                log('[Testing] wrote %s' % self.args.results_file)
            log('[Testing] no successful attacks found, no plots')
            return

        perturbation_images = numpy.repeat(self.perturbation_images, num_attempts, axis=0)
        perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)

        #
        # Compute nearest neighbors for perturbations and test images,
        # to backproject them into the latent space.
        # Also compute the dot product betweenm perturbations and a local
        # plane approximation base don the three nearest neighbors.
        #

        log('[Testing] computing nearest neighbor ...')
        nearest_neighbors_indices = self.compute_nearest_neighbors(perturbation_images)
        nearest_neighbors = self.train_theta[nearest_neighbors_indices[:, 0]]
        perturbation_nearest_neighbor_indices = self.compute_nearest_neighbors(perturbations)
        perturbation_nearest_neighbor = self.train_theta[perturbation_nearest_neighbor_indices[:, 0]]

        # Compute statistics over the perturbation with respect to the plane
        # defined by the three nearest neighbors of the corresponding test sample.
        if self.args.plot_directory and self.args.plot_manifolds and utils.display():
            pure_perturbations = perturbations[raw_overall_success] - perturbation_images[raw_overall_success]
            pure_perturbations_norm = numpy.linalg.norm(pure_perturbations, ord=2, axis=1)
            for k in range(10):
                direction = perturbation_images[raw_overall_success] - self.train_images[nearest_neighbors_indices[:, k][raw_overall_success]]
                direction_norm = numpy.linalg.norm(direction, ord=2, axis=1)
                dot_products = numpy.einsum('ij,ij->i', direction, pure_perturbations)
                dot_product_norms = numpy.multiply(pure_perturbations_norm, direction_norm)
                dot_product_norms[dot_product_norms == 0] = 1
                dot_products /= dot_product_norms
                dot_products = numpy.degrees(numpy.arccos(dot_products))

                # matplotlib's hsitogram plots give weird error if there are NaN values, so simple check:
                if dot_products.shape[0] > 0 and not numpy.any(dot_products != dot_products):
                    plot_file = os.path.join(self.args.plot_directory, 'dot_products_nn%d' % k)
                    plot.histogram(plot_file, dot_products, 100,
                                   title='Dot Products Between Adversarial Perturbations and Direction to Nearest Neighbor %d' % k,
                                   xlabel='Dot Product (Between Normalized Vectors)', ylabel='Count')
                    log('[Testing] wrote %s' % plot_file)

        #
        # We compute some simple statistics:
        # - raw success rate: fraction of successful attack without considering epsilon
        # - corrected success rate: fraction of successful attacks within epsilon-ball
        # - raw average perturbation: average distance to original samples (for successful attacks)
        # - corrected average perturbation: average distance to original samples for perturbations
        #   within epsilon-ball (for successful attacks).
        # These statistics can also be computed per class.
        # And these statistics are computed with respect to three norms.

        if self.args.plot_directory and utils.display():
            iterations = success[raw_overall_success]
            x = numpy.arange(numpy.max(iterations) + 1)
            y = numpy.bincount(iterations)
            plot_file = os.path.join(self.args.plot_directory, 'iterations')
            plot.bar(plot_file, x, y,
                     title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count')
            log('[Testing] wrote %s' % plot_file)

        for n in range(len(self.norms)):
            norm = self.norms[n]
            delta = numpy.linalg.norm(perturbation_images - perturbations, norm, axis=1)
            latent_delta = numpy.linalg.norm(nearest_neighbors - perturbation_nearest_neighbor, norm, axis=1)

            if self.args.plot_directory and utils.display():
                plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm)
                plot.histogram(plot_file, delta[raw_overall_success], 50, title='Distribution of $L_{%g}$ Distances of Successful Attacks' % norm,
                               xlabel='Distance', ylabel='Count')
                log('[Testing] wrote %s' % plot_file)

            #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0]
            #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0)
            #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy)
            #log('[Testing] attacked model accuracy: %g' % debug_accuracy)
            #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
            #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)

            N_accuracy = numpy.sum(accuracy)
            self.results[n]['raw_success'] = numpy.sum(raw_overall_success) / N_accuracy
            self.results[n]['raw_iteration'] = numpy.average(success[raw_overall_success])
            self.results[n]['raw_average'] = numpy.average(delta[raw_overall_success]) if numpy.any(raw_overall_success) else 0
            self.results[n]['raw_latent'] = numpy.average(latent_delta[raw_overall_success]) if numpy.any(raw_overall_success) else 0

            raw_class_success = numpy.zeros((N_class, perturbation_images.shape[0]), bool)
            self.results[n]['raw_class_success'] = numpy.zeros((N_class))
            self.results[n]['raw_class_average'] = numpy.zeros((N_class))
            self.results[n]['raw_class_latent'] = numpy.zeros((N_class))

            for c in range(N_class):
                N_samples = numpy.sum(numpy.logical_and(accuracy, perturbation_codes == c))
                if N_samples <= 0:
                    continue;

                raw_class_success[c] = numpy.logical_and(raw_overall_success, perturbation_codes == c)
                self.results[n]['raw_class_success'][c] = numpy.sum(raw_class_success[c]) / N_samples
                if numpy.any(raw_class_success[c]):
                    self.results[n]['raw_class_average'][c] = numpy.average(delta[raw_class_success[c].astype(bool)])
                if numpy.any(raw_class_success[c]):
                    self.results[n]['raw_class_latent'][c] = numpy.average(latent_delta[raw_class_success[c].astype(bool)])

        if self.args.results_file:
            utils.write_pickle(self.args.results_file, self.results)
            log('[Testing] wrote %s' % self.args.results_file)

    def plot_manifolds(self):
        """
        Plot manifolds.
        """

        fit = self.test_images.shape[0] // 100
        N_class = numpy.max(self.test_codes) + 1
        num_attempts = self.perturbations.shape[0]
        success = numpy.swapaxes(self.success, 0, 1)
        success = success.reshape((success.shape[0] * success.shape[1]))
        perturbations = self.perturbations.reshape((self.perturbations.shape[0] * self.perturbations.shape[1], -1))
        perturbations = perturbations[success >= 0]
        codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)
        codes = codes[success >= 0]

        #
        # Plot all classes and adversarial examples in image space for individual classes as well as all classes.
        #

        manifold_visualization = plot.ManifoldVisualization('tsne', pre_pca=40)
        manifold_visualization.fit(self.test_images[:fit])
        log('[Testing] computed t-SNE on test images')

        for n in range(N_class):
            labels = ['Class %d' % (nn + 1) for nn in range(N_class)] + ['Adversarial Examples Class %d' % (n + 1)]
            data = numpy.concatenate((
                self.test_images[:fit],
                perturbations[codes == n]
            ))
            classes = numpy.concatenate((
                self.test_codes[:fit],
                numpy.ones((perturbations[codes == n].shape[0])) * N_class,
            ))
            plot_file = os.path.join(self.args.plot_directory, 'perturbations_%d' % (n + 1))
            manifold_visualization.visualize(plot_file, data, classes, labels,
                                             title='Adversarial Examples Class %d\n(The adversarial examples are projected into the embedding using learned SVRs)' % n)
            log('[Testing] wrote %s' % plot_file)

        labels = ['Class %d' % (n + 1) for n in range(N_class)] + ['Adversarial Examples Class %d' % (n + 1) for n in
                                                                   range(N_class)]
        data = numpy.concatenate((
            self.test_images[:fit],
            perturbations
        ))
        classes = numpy.concatenate((
            self.test_codes[:fit],
            codes + N_class,
        ))
        plot_file = os.path.join(self.args.plot_directory, 'perturbations')
        manifold_visualization.visualize(plot_file, data, classes, labels,
                                         title='Adversarial Examples\n(The adversarial examples are projected into the embedding using learned SVRs)')
        log('[Testing] wrote %s' % plot_file)

        #
        # Plot all classes and adversarial examples in latent space.
        #

        perturbation_neighbors_indices = self.compute_nearest_neighbors(perturbations)
        perturbation_neighbors = self.train_theta[perturbation_neighbors_indices[:, 0]]
        perturbation_neighbors = numpy.squeeze(perturbation_neighbors)
        perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)
        perturbation_neighbors = numpy.concatenate((perturbation_neighbors[success >= 0], perturbation_codes[success >= 0]), axis=1)
        plot_theta = numpy.concatenate((self.test_theta, self.test_codes), axis=1)
        manifold_visualization = plot.ManifoldVisualization('tsne', pre_pca=None)
        manifold_visualization.fit(plot_theta[:fit])
        log('[Testing] computed t-SNE on test codes')

        for c in range(N_class):
            labels = ['Class %d' % (cc + 1) for cc in range(N_class)] + ['Adversarial Examples Class %d' % (c + 1)]
            data = numpy.concatenate((
                plot_theta[:fit],
                perturbation_neighbors[codes == c]
            ))
            classes = numpy.concatenate((
                self.test_codes[:fit],
                numpy.ones((perturbation_neighbors[codes == c].shape[0])) * N_class,
            ))
            plot_file = os.path.join(self.args.plot_directory, 'latent_perturbations_%d' % (c + 1))
            manifold_visualization.visualize(plot_file, data, classes, labels,
                                             title='Adversarial Examples Class %d\n(The adversarial examples are projected into the embedding using learned SVRs)' % c)
            log('[Testing] wrote %s' % plot_file)

        labels = ['Class %d' % (c + 1) for c in range(N_class)] + ['Adversarial Examples Class %d' % (c + 1) for c in range(N_class)]
        data = numpy.concatenate((
            plot_theta[:fit],
            perturbation_neighbors
        ))
        classes = numpy.concatenate((
            self.test_codes[:fit],
            codes + N_class,
        ))
        plot_file = os.path.join(self.args.plot_directory, 'latent_perturbations')
        manifold_visualization.visualize(plot_file, data, classes, labels,
                                         title='Adversarial Examples\n(The adversarial examples are projected into the embedding using learned SVRs)')
        log('[Testing] wrote %s' % plot_file)

    def print_statistics(self):
        """
        Print statistics.
        """

        N_class = numpy.max(self.test_codes) + 1
        table_data = []

        for n in range(len(self.norms)):
            table_headings = ['(Class) Norm', 'Metric, Epsilon', 'Raw']
            table_data.append(table_headings)
            norm = self.norms[n]

            table_row = ['    L_%.3g' % norm, 'Success Rate', '%.3g' % self.results[n]['raw_success']]
            table_data.append(table_row)

            table_row = ['    L_%.3g' % norm, 'Image Distance', '%.3g' % self.results[n]['raw_average']]
            table_data.append(table_row)

            table_row = ['    L_%.3g' % norm, 'Latent Distance', '%.3g' % self.results[n]['raw_latent']]
            table_data.append(table_row)

            for c in range(N_class):
                table_row = ['(%d) L_%.3g' % (c, norm), 'Success Rate', '%.3g' % self.results[n]['raw_class_success'][c]]
                table_data.append(table_row)

                table_row = ['(%d) L_%.3g' % (c, norm), 'Image Distance', '%.3g' % self.results[n]['raw_class_average'][c]]
                table_data.append(table_row)

                table_row = ['(%d) L_%.3g' % (c, norm), 'Latent Distance', '%.3g' % self.results[n]['raw_class_latent'][c]]
                table_data.append(table_row)
            table_data.append(['---']*2)

        table = terminaltables.AsciiTable(table_data)
        log(table.table)

    def main(self):
        """
        Main.
        """

        self.load_data()
        if self.args.latent:
            self.compute_latent_statistics()
        else:
            self.compute_statistics()
        self.print_statistics()
        if self.args.plot_directory and self.args.plot_manifolds and utils.display():
            self.plot_manifolds()


if __name__ == '__main__':
    program = TestAttackClassifier()
    program.main()

import os
import sys

sys.path.insert(1, os.path.dirname(os.path.realpath(__file__)) + '/../')
from common import utils
from common.log import log
from common import paths

import numpy
import argparse
import sklearn.decomposition
import sklearn.neighbors
import terminaltables


if utils.display():
    from common import plot


class TestAttackClassifier:
    """
    Test a trained classifier.
    """

    def __init__(self, args=None):
        """
        Initialize.

        :param args: optional arguments if not to use sys.argv
        :type args: [str]
        """

        self.args = None
        """ Arguments of program. """

        parser = self.get_parser()
        if args is not None:
            self.args = parser.parse_args(args)
        else:
            self.args = parser.parse_args()

        self.test_images = None
        """ (numpy.ndarray) Images to test on. """

        self.perturbation_images = None
        """ (numpy.ndarray) Images corresponding to perturbations. """

        self.train_images = None
        """ (numpy.ndarray) Images to train on. """

        self.test_theta = None
        """ (numpy.ndarray) Transformation parameters. """

        self.train_theta = None
        """ (numpy.ndarra) Transformation parameters, i.e. latent codes. """

        self.perturbations = None
        """ (numpy.ndarray) Perturbations per test image. """

        self.test_codes = None
        """ (numpy.ndarray) Test codes."""

        self.perturbation_codes = None
        """ (numpy.ndarray) Codes corresponding to perturbations. """

        self.accuracy = None
        """ (numpy.ndarray) Success indicator of test samples. """

        self.success = None
        """ (numpy.ndarray) Success indicator for perturbations."""

        self.pca = None
        """ (sklearn.decomposition.IncrementalPCA) PCA to make nearest neighbor more efficient. """

        self.neighbors = None
        """ (sklearn.neighbors.NearestNeighbors) Nearest neighbor model. """

        self.norms = [1, 2, float('inf')]
        """ ([float]) Norms to evaluate. """

        self.results = []
        """ (dict) Dictionary containing all statistics. """

        for n in range(len(self.norms)):
            self.results.append(dict())

        log('-- ' + self.__class__.__name__)
        for key in vars(self.args):
            log('[Testing] %s=%s' % (key, str(getattr(self.args, key))))

    def get_parser(self):
        """
        Get parser.

        :return: parser
        :rtype: argparse.ArgumentParser
        """

        parser = argparse.ArgumentParser(description='Test attacks on classifier.')
        parser.add_argument('-test_images_file', default=paths.test_images_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-train_images_file', default=paths.train_images_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-test_theta_file', default=paths.test_theta_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-train_theta_file', default=paths.train_theta_file(), help='HDF5 file containing dataset.', type=str)
        parser.add_argument('-test_codes_file', default=paths.test_codes_file(), help='HDF5 file containing labels.', type=str)
        parser.add_argument('-label_index', default=2, help='Column index in label file.', type=int)
        parser.add_argument('-accuracy_file', default=paths.results_file('classifier/accuracy'), help='Correctly classified test samples of classifier.', type=str)
        parser.add_argument('-perturbations_file', default=paths.results_file('classifier/perturbations'), help='HDF5 file containing perturbations.', type=str)
        parser.add_argument('-success_file', default=paths.results_file('classifier/success'), help='HDF5 file indicating attack success.', type=str)
        parser.add_argument('-results_file', default='', help='Path to pickled results file.', type=str)
        parser.add_argument('-plot_directory', default=paths.experiment_dir('classifier'), help='Path to PNG plot file for success rate.', type=str)
        parser.add_argument('-plot_manifolds', default=False, action='store_true', help='Whether to plot manifolds.')
        parser.add_argument('-latent', default=False, action='store_true', help='Latent statistics.')

        return parser

    def load_data(self):
        """
        Load data and model.
        """

        self.test_images = utils.read_hdf5(self.args.test_images_file)
        self.test_images = self.test_images.reshape((self.test_images.shape[0], -1))
        log('[Testing] read %s' % self.args.test_images_file)

        self.train_images = utils.read_hdf5(self.args.train_images_file)
        self.train_images = self.train_images.reshape((self.train_images.shape[0], -1))
        log('[Testing] read %s' % self.args.train_images_file)

        self.train_theta = utils.read_hdf5(self.args.train_theta_file)
        log('[Testing] read %s' % self.args.train_theta_file)

        self.test_theta = utils.read_hdf5(self.args.test_theta_file)
        log('[Testing] read %s' % self.args.test_theta_file)

        self.test_codes = utils.read_hdf5(self.args.test_codes_file)
        self.test_codes = self.test_codes[:, self.args.label_index]
        log('[Testing] read %s' % self.args.test_codes_file)

        self.perturbations = utils.read_hdf5(self.args.perturbations_file)
        if len(self.perturbations.shape) > 3:
            self.perturbations = self.perturbations.reshape((self.perturbations.shape[0], self.perturbations.shape[1], -1))
        self.perturbation_images = self.test_images[:self.perturbations.shape[1]]
        self.perturbation_codes = self.test_codes[:self.perturbations.shape[1]]
        log('[Testing] read %s' % self.args.perturbations_file)
        assert not numpy.any(self.perturbations != self.perturbations), 'NaN in perturbations'

        self.success = utils.read_hdf5(self.args.success_file)
        log('[Testing] read %s' % self.args.success_file)

        self.accuracy = utils.read_hdf5(self.args.accuracy_file)
        self.accuracy = self.accuracy[:self.perturbations.shape[1]]
        log('[Testing] read %s' % self.args.accuracy_file)

    def compute_nearest_neighbors(self, images):
        """
        Compute distances in image and latent space.

        :param images: images to get nearest neighbors for
        :type images: numpy.ndarray
        :param norm: norm to use
        :type norm: float
        """

        fit = 100000
        if self.pca is None:
            self.pca = sklearn.decomposition.IncrementalPCA(n_components=20)
            self.pca.fit(self.train_images[:fit])
            log('[Testing] fitted PCA')
        if self.neighbors is None:
            data = self.pca.transform(self.train_images)
            self.neighbors = sklearn.neighbors.NearestNeighbors(n_neighbors=10, algorithm='kd_tree')
            self.neighbors.fit(data[:fit])
            log('[Testing] fitted nearest neighbor')

        data = self.pca.transform(images)
        _, indices = self.neighbors.kneighbors(data)
        return indices

    def compute_statistics(self):
        """
        Compute statistics based on distances.
        """

        N_class = numpy.max(self.test_codes) + 1
        num_attempts = self.perturbations.shape[0]

        perturbations = numpy.swapaxes(self.perturbations, 0, 1)
        perturbations = perturbations.reshape((perturbations.shape[0]*perturbations.shape[1], perturbations.shape[2]))
        success = numpy.swapaxes(self.success, 0, 1)
        success = success.reshape((success.shape[0]*success.shape[1]))

        accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0)
        # Raw success is the base for all statistics, as we need to consider only these
        # attacks that are successful and where the classifier originally was correct.
        raw_overall_success = numpy.logical_and(success >= 0, accuracy)
        log('[Testing] %d valid attacks' % numpy.sum(raw_overall_success))

        # For off-manifold attacks this should not happen, but save is save.
        if not numpy.any(raw_overall_success):
            for n in range(len(self.norms)):
                for type in ['raw_success', 'raw_iteration', 'raw_average', 'raw_latent']:
                    self.results[n][type] = 0
                for type in ['raw_class_success', 'raw_class_average', 'raw_class_latent']:
                    self.results[n][type] = numpy.zeros((N_class))
            if self.args.results_file:
                utils.write_pickle(self.args.results_file, self.results)
                log('[Testing] wrote %s' % self.args.results_file)
            log('[Testing] no successful attacks found, no plots')
            return

        perturbation_images = numpy.repeat(self.perturbation_images, num_attempts, axis=0)
        perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)

        #
        # We compute some simple statistics:
        # - raw success rate: fraction of successful attack without considering epsilon
        # - corrected success rate: fraction of successful attacks within epsilon-ball
        # - raw average perturbation: average distance to original samples (for successful attacks)
        # - corrected average perturbation: average distance to original samples for perturbations
        #   within epsilon-ball (for successful attacks).
        # These statistics can also be computed per class.
        # And these statistics are computed with respect to three norms.

        if self.args.plot_directory and utils.display():
            iterations = success[raw_overall_success]
            x = numpy.arange(numpy.max(iterations) + 1)
            y = numpy.bincount(iterations)
            plot_file = os.path.join(self.args.plot_directory, 'iterations')
            plot.bar(plot_file, x, y, title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count')
            log('[Testing] wrote %s' % plot_file)

        for n in range(len(self.norms)):
            norm = self.norms[n]
            delta = numpy.linalg.norm(perturbation_images - perturbations, norm, axis=1)

            if self.args.plot_directory and utils.display():
                plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm)
                plot.histogram(plot_file, delta[raw_overall_success], 50, title='Distribution of $L_{%g}$ Distances of Successful Attacks' % norm,
                              xlabel='Distance', ylabel='Count')
                log('[Testing] wrote %s' % plot_file)

            #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0]
            #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0)
            #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy)
            #log('[Testing] attacked model accuracy: %g' % debug_accuracy)
            #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
            #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)

            N_accuracy = numpy.sum(accuracy)
            self.results[n]['raw_success'] = numpy.sum(raw_overall_success) / N_accuracy
            self.results[n]['raw_iteration'] = numpy.average(success[raw_overall_success])
            self.results[n]['raw_average'] = numpy.average(delta[raw_overall_success]) if numpy.any(raw_overall_success) else 0
            self.results[n]['raw_latent'] = 0

            raw_class_success = numpy.zeros((N_class, perturbation_images.shape[0]), bool)
            self.results[n]['raw_class_success'] = numpy.zeros((N_class))
            self.results[n]['raw_class_average'] = numpy.zeros((N_class))
            self.results[n]['raw_class_latent'] = numpy.zeros((N_class))

            for c in range(N_class):
                N_samples = numpy.sum(numpy.logical_and(accuracy, perturbation_codes == c))
                if N_samples <= 0:
                    continue;

                raw_class_success[c] = numpy.logical_and(raw_overall_success, perturbation_codes == c)
                self.results[n]['raw_class_success'][c] = numpy.sum(raw_class_success[c]) / N_samples
                if numpy.any(raw_class_success[c]):
                    self.results[n]['raw_class_average'][c] = numpy.average(delta[raw_class_success[c].astype(bool)])

        if self.args.results_file:
            utils.write_pickle(self.args.results_file, self.results)
            log('[Testing] wrote %s' % self.args.results_file)

    def compute_latent_statistics(self):
        """
        Compute latent statistics.
        """

        N_class = numpy.max(self.test_codes) + 1
        num_attempts = self.perturbations.shape[0]

        perturbations = numpy.swapaxes(self.perturbations, 0, 1)
        perturbations = perturbations.reshape((perturbations.shape[0] * perturbations.shape[1], perturbations.shape[2]))
        success = numpy.swapaxes(self.success, 0, 1)
        success = success.reshape((success.shape[0] * success.shape[1]))

        accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0)
        # Raw success is the base for all statistics, as we need to consider only these
        # attacks that are successful and where the classifier originally was correct.
        raw_overall_success = numpy.logical_and(success >= 0, accuracy)

        # For off-manifold attacks this should not happen, but save is save.
        if not numpy.any(raw_overall_success):
            for n in range(len(self.norms)):
                for type in ['raw_success', 'raw_iteration', 'raw_average', 'raw_latent']:
                    self.results[n][type] = 0
                for type in ['raw_class_success', 'raw_class_average', 'raw_class_latent']:
                    self.results[n][type] = numpy.zeros((N_class))
            if self.args.results_file:
                utils.write_pickle(self.args.results_file, self.results)
                log('[Testing] wrote %s' % self.args.results_file)
            log('[Testing] no successful attacks found, no plots')
            return

        perturbation_images = numpy.repeat(self.perturbation_images, num_attempts, axis=0)
        perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)

        #
        # Compute nearest neighbors for perturbations and test images,
        # to backproject them into the latent space.
        # Also compute the dot product betweenm perturbations and a local
        # plane approximation base don the three nearest neighbors.
        #

        log('[Testing] computing nearest neighbor ...')
        nearest_neighbors_indices = self.compute_nearest_neighbors(perturbation_images)
        nearest_neighbors = self.train_theta[nearest_neighbors_indices[:, 0]]
        perturbation_nearest_neighbor_indices = self.compute_nearest_neighbors(perturbations)
        perturbation_nearest_neighbor = self.train_theta[perturbation_nearest_neighbor_indices[:, 0]]

        # Compute statistics over the perturbation with respect to the plane
        # defined by the three nearest neighbors of the corresponding test sample.
        if self.args.plot_directory and self.args.plot_manifolds and utils.display():
            pure_perturbations = perturbations[raw_overall_success] - perturbation_images[raw_overall_success]
            pure_perturbations_norm = numpy.linalg.norm(pure_perturbations, ord=2, axis=1)
            for k in range(10):
                direction = perturbation_images[raw_overall_success] - self.train_images[nearest_neighbors_indices[:, k][raw_overall_success]]
                direction_norm = numpy.linalg.norm(direction, ord=2, axis=1)
                dot_products = numpy.einsum('ij,ij->i', direction, pure_perturbations)
                dot_product_norms = numpy.multiply(pure_perturbations_norm, direction_norm)
                dot_product_norms[dot_product_norms == 0] = 1
                dot_products /= dot_product_norms
                dot_products = numpy.degrees(numpy.arccos(dot_products))

                # matplotlib's hsitogram plots give weird error if there are NaN values, so simple check:
                if dot_products.shape[0] > 0 and not numpy.any(dot_products != dot_products):
                    plot_file = os.path.join(self.args.plot_directory, 'dot_products_nn%d' % k)
                    plot.histogram(plot_file, dot_products, 100,
                                   title='Dot Products Between Adversarial Perturbations and Direction to Nearest Neighbor %d' % k,
                                   xlabel='Dot Product (Between Normalized Vectors)', ylabel='Count')
                    log('[Testing] wrote %s' % plot_file)

        #
        # We compute some simple statistics:
        # - raw success rate: fraction of successful attack without considering epsilon
        # - corrected success rate: fraction of successful attacks within epsilon-ball
        # - raw average perturbation: average distance to original samples (for successful attacks)
        # - corrected average perturbation: average distance to original samples for perturbations
        #   within epsilon-ball (for successful attacks).
        # These statistics can also be computed per class.
        # And these statistics are computed with respect to three norms.

        if self.args.plot_directory and utils.display():
            iterations = success[raw_overall_success]
            x = numpy.arange(numpy.max(iterations) + 1)
            y = numpy.bincount(iterations)
            plot_file = os.path.join(self.args.plot_directory, 'iterations')
            plot.bar(plot_file, x, y,
                     title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count')
            log('[Testing] wrote %s' % plot_file)

        for n in range(len(self.norms)):
            norm = self.norms[n]
            delta = numpy.linalg.norm(perturbation_images - perturbations, norm, axis=1)
            latent_delta = numpy.linalg.norm(nearest_neighbors - perturbation_nearest_neighbor, norm, axis=1)

            if self.args.plot_directory and utils.display():
                plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm)
                plot.histogram(plot_file, delta[raw_overall_success], 50, title='Distribution of $L_{%g}$ Distances of Successful Attacks' % norm,
                               xlabel='Distance', ylabel='Count')
                log('[Testing] wrote %s' % plot_file)

            #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0]
            #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0)
            #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy)
            #log('[Testing] attacked model accuracy: %g' % debug_accuracy)
            #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
            #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)

            N_accuracy = numpy.sum(accuracy)
            self.results[n]['raw_success'] = numpy.sum(raw_overall_success) / N_accuracy
            self.results[n]['raw_iteration'] = numpy.average(success[raw_overall_success])
            self.results[n]['raw_average'] = numpy.average(delta[raw_overall_success]) if numpy.any(raw_overall_success) else 0
            self.results[n]['raw_latent'] = numpy.average(latent_delta[raw_overall_success]) if numpy.any(raw_overall_success) else 0

            raw_class_success = numpy.zeros((N_class, perturbation_images.shape[0]), bool)
            self.results[n]['raw_class_success'] = numpy.zeros((N_class))
            self.results[n]['raw_class_average'] = numpy.zeros((N_class))
            self.results[n]['raw_class_latent'] = numpy.zeros((N_class))

            for c in range(N_class):
                N_samples = numpy.sum(numpy.logical_and(accuracy, perturbation_codes == c))
                if N_samples <= 0:
                    continue;

                raw_class_success[c] = numpy.logical_and(raw_overall_success, perturbation_codes == c)
                self.results[n]['raw_class_success'][c] = numpy.sum(raw_class_success[c]) / N_samples
                if numpy.any(raw_class_success[c]):
                    self.results[n]['raw_class_average'][c] = numpy.average(delta[raw_class_success[c].astype(bool)])
                if numpy.any(raw_class_success[c]):
                    self.results[n]['raw_class_latent'][c] = numpy.average(latent_delta[raw_class_success[c].astype(bool)])

        if self.args.results_file:
            utils.write_pickle(self.args.results_file, self.results)
            log('[Testing] wrote %s' % self.args.results_file)

    def plot_manifolds(self):
        """
        Plot manifolds.
        """

        fit = self.test_images.shape[0] // 100
        N_class = numpy.max(self.test_codes) + 1
        num_attempts = self.perturbations.shape[0]
        success = numpy.swapaxes(self.success, 0, 1)
        success = success.reshape((success.shape[0] * success.shape[1]))
        perturbations = self.perturbations.reshape((self.perturbations.shape[0] * self.perturbations.shape[1], -1))
        perturbations = perturbations[success >= 0]
        codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)
        codes = codes[success >= 0]

        #
        # Plot all classes and adversarial examples in image space for individual classes as well as all classes.
        #

        manifold_visualization = plot.ManifoldVisualization('tsne', pre_pca=40)
        manifold_visualization.fit(self.test_images[:fit])
        log('[Testing] computed t-SNE on test images')

        for n in range(N_class):
            labels = ['Class %d' % (nn + 1) for nn in range(N_class)] + ['Adversarial Examples Class %d' % (n + 1)]
            data = numpy.concatenate((
                self.test_images[:fit],
                perturbations[codes == n]
            ))
            classes = numpy.concatenate((
                self.test_codes[:fit],
                numpy.ones((perturbations[codes == n].shape[0])) * N_class,
            ))
            plot_file = os.path.join(self.args.plot_directory, 'perturbations_%d' % (n + 1))
            manifold_visualization.visualize(plot_file, data, classes, labels,
                                             title='Adversarial Examples Class %d\n(The adversarial examples are projected into the embedding using learned SVRs)' % n)
            log('[Testing] wrote %s' % plot_file)

        labels = ['Class %d' % (n + 1) for n in range(N_class)] + ['Adversarial Examples Class %d' % (n + 1) for n in
                                                                   range(N_class)]
        data = numpy.concatenate((
            self.test_images[:fit],
            perturbations
        ))
        classes = numpy.concatenate((
            self.test_codes[:fit],
            codes + N_class,
        ))
        plot_file = os.path.join(self.args.plot_directory, 'perturbations')
        manifold_visualization.visualize(plot_file, data, classes, labels,
                                         title='Adversarial Examples\n(The adversarial examples are projected into the embedding using learned SVRs)')
        log('[Testing] wrote %s' % plot_file)

        #
        # Plot all classes and adversarial examples in latent space.
        #

        perturbation_neighbors_indices = self.compute_nearest_neighbors(perturbations)
        perturbation_neighbors = self.train_theta[perturbation_neighbors_indices[:, 0]]
        perturbation_neighbors = numpy.squeeze(perturbation_neighbors)
        perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0)
        perturbation_neighbors = numpy.concatenate((perturbation_neighbors[success >= 0], perturbation_codes[success >= 0]), axis=1)
        plot_theta = numpy.concatenate((self.test_theta, self.test_codes), axis=1)
        manifold_visualization = plot.ManifoldVisualization('tsne', pre_pca=None)
        manifold_visualization.fit(plot_theta[:fit])
        log('[Testing] computed t-SNE on test codes')

        for c in range(N_class):
            labels = ['Class %d' % (cc + 1) for cc in range(N_class)] + ['Adversarial Examples Class %d' % (c + 1)]
            data = numpy.concatenate((
                plot_theta[:fit],
                perturbation_neighbors[codes == c]
            ))
            classes = numpy.concatenate((
                self.test_codes[:fit],
                numpy.ones((perturbation_neighbors[codes == c].shape[0])) * N_class,
            ))
            plot_file = os.path.join(self.args.plot_directory, 'latent_perturbations_%d' % (c + 1))
            manifold_visualization.visualize(plot_file, data, classes, labels,
                                             title='Adversarial Examples Class %d\n(The adversarial examples are projected into the embedding using learned SVRs)' % c)
            log('[Testing] wrote %s' % plot_file)

        labels = ['Class %d' % (c + 1) for c in range(N_class)] + ['Adversarial Examples Class %d' % (c + 1) for c in range(N_class)]
        data = numpy.concatenate((
            plot_theta[:fit],
            perturbation_neighbors
        ))
        classes = numpy.concatenate((
            self.test_codes[:fit],
            codes + N_class,
        ))
        plot_file = os.path.join(self.args.plot_directory, 'latent_perturbations')
        manifold_visualization.visualize(plot_file, data, classes, labels,
                                         title='Adversarial Examples\n(The adversarial examples are projected into the embedding using learned SVRs)')
        log('[Testing] wrote %s' % plot_file)

    def print_statistics(self):
        """
        Print statistics.
        """

        N_class = numpy.max(self.test_codes) + 1
        table_data = []

        for n in range(len(self.norms)):
            table_headings = ['(Class) Norm', 'Metric, Epsilon', 'Raw']
            table_data.append(table_headings)
            norm = self.norms[n]

            table_row = ['    L_%.3g' % norm, 'Success Rate', '%.3g' % self.results[n]['raw_success']]
            table_data.append(table_row)

            table_row = ['    L_%.3g' % norm, 'Image Distance', '%.3g' % self.results[n]['raw_average']]
            table_data.append(table_row)

            table_row = ['    L_%.3g' % norm, 'Latent Distance', '%.3g' % self.results[n]['raw_latent']]
            table_data.append(table_row)

            for c in range(N_class):
                table_row = ['(%d) L_%.3g' % (c, norm), 'Success Rate', '%.3g' % self.results[n]['raw_class_success'][c]]
                table_data.append(table_row)

                table_row = ['(%d) L_%.3g' % (c, norm), 'Image Distance', '%.3g' % self.results[n]['raw_class_average'][c]]
                table_data.append(table_row)

                table_row = ['(%d) L_%.3g' % (c, norm), 'Latent Distance', '%.3g' % self.results[n]['raw_class_latent'][c]]
                table_data.append(table_row)
            table_data.append(['---']*2)

        table = terminaltables.AsciiTable(table_data)
        log(table.table)

    def main(self):
        """
        Main.
        """

        self.load_data()
        if self.args.latent:
            self.compute_latent_statistics()
        else:
            self.compute_statistics()
        self.print_statistics()
        if self.args.plot_directory and self.args.plot_manifolds and utils.display():
            self.plot_manifolds()


if __name__ == '__main__':
    program = TestAttackClassifier()
    program.main()
Test a trained classifier. Initialize.          :param args: optional arguments if not to use sys.argv         :type args: [str] Arguments of program. (numpy.ndarray) Images to test on. (numpy.ndarray) Images corresponding to perturbations. (numpy.ndarray) Images to train on. (numpy.ndarray) Transformation parameters. (numpy.ndarra) Transformation parameters, i.e. latent codes. (numpy.ndarray) Perturbations per test image. (numpy.ndarray) Test codes. (numpy.ndarray) Codes corresponding to perturbations. (numpy.ndarray) Success indicator of test samples. (numpy.ndarray) Success indicator for perturbations. (sklearn.decomposition.IncrementalPCA) PCA to make nearest neighbor more efficient. (sklearn.neighbors.NearestNeighbors) Nearest neighbor model. ([float]) Norms to evaluate. (dict) Dictionary containing all statistics. Get parser.          :return: parser         :rtype: argparse.ArgumentParser Load data and model. Compute distances in image and latent space.          :param images: images to get nearest neighbors for         :type images: numpy.ndarray         :param norm: norm to use         :type norm: float Compute statistics based on distances. # Raw success is the base for all statistics, as we need to consider only these # attacks that are successful and where the classifier originally was correct. # For off-manifold attacks this should not happen, but save is save. # # We compute some simple statistics: # - raw success rate: fraction of successful attack without considering epsilon # - corrected success rate: fraction of successful attacks within epsilon-ball # - raw average perturbation: average distance to original samples (for successful attacks) # - corrected average perturbation: average distance to original samples for perturbations #   within epsilon-ball (for successful attacks). # These statistics can also be computed per class. # And these statistics are computed with respect to three norms. #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0] #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0) #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy) #log('[Testing] attacked model accuracy: %g' % debug_accuracy) #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction) #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction) Compute latent statistics. # Raw success is the base for all statistics, as we need to consider only these # attacks that are successful and where the classifier originally was correct. # For off-manifold attacks this should not happen, but save is save. # # Compute nearest neighbors for perturbations and test images, # to backproject them into the latent space. # Also compute the dot product betweenm perturbations and a local # plane approximation base don the three nearest neighbors. # # Compute statistics over the perturbation with respect to the plane # defined by the three nearest neighbors of the corresponding test sample. # matplotlib's hsitogram plots give weird error if there are NaN values, so simple check: # # We compute some simple statistics: # - raw success rate: fraction of successful attack without considering epsilon # - corrected success rate: fraction of successful attacks within epsilon-ball # - raw average perturbation: average distance to original samples (for successful attacks) # - corrected average perturbation: average distance to original samples for perturbations #   within epsilon-ball (for successful attacks). # These statistics can also be computed per class. # And these statistics are computed with respect to three norms. #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0] #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0) #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy) #log('[Testing] attacked model accuracy: %g' % debug_accuracy) #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction) #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction) Plot manifolds. # # Plot all classes and adversarial examples in image space for individual classes as well as all classes. # # # Plot all classes and adversarial examples in latent space. # Print statistics. Main.