Load a saved model, gather its predictions, and save a confidence report.


  This function works by running a single MaxConfidence attack on each example,
  using SPSA as the underyling optimizer.
  This is not intended to be a strong generic attack.
  It is intended to be a test to uncover gradient masking.

  :param filepath: path to model to evaluate
  :param train_start: index of first training set example to use
  :param train_end: index of last training set example to use
  :param test_start: index of first test set example to use
  :param test_end: index of last test set example to use
  :param batch_size: size of evaluation batches
  :param which_set: 'train' or 'test'
  :param nb_iter: Number of iterations of PGD to run per class
  :param spsa_samples: Number of samples for SPSA

def make_confidence_report_spsa(filepath, train_start=TRAIN_START,
                                train_end=TRAIN_END,
                                test_start=TEST_START, test_end=TEST_END,
                                batch_size=BATCH_SIZE, which_set=WHICH_SET,
                                report_path=REPORT_PATH,
                                nb_iter=NB_ITER_SPSA,
                                spsa_samples=SPSA_SAMPLES,
                                spsa_iters=SPSA.DEFAULT_SPSA_ITERS):
  """
  Load a saved model, gather its predictions, and save a confidence report.


  This function works by running a single MaxConfidence attack on each example,
  using SPSA as the underyling optimizer.
  This is not intended to be a strong generic attack.
  It is intended to be a test to uncover gradient masking.

  :param filepath: path to model to evaluate
  :param train_start: index of first training set example to use
  :param train_end: index of last training set example to use
  :param test_start: index of first test set example to use
  :param test_end: index of last test set example to use
  :param batch_size: size of evaluation batches
  :param which_set: 'train' or 'test'
  :param nb_iter: Number of iterations of PGD to run per class
  :param spsa_samples: Number of samples for SPSA
  """

  # Set TF random seed to improve reproducibility
  tf.set_random_seed(1234)

  # Set logging level to see debug information
  set_log_level(logging.INFO)

  # Create TF session
  sess = tf.Session()

  if report_path is None:
    assert filepath.endswith('.joblib')
    report_path = filepath[:-len('.joblib')] + "_spsa_report.joblib"

  with sess.as_default():
    model = load(filepath)
  assert len(model.get_params()) > 0
  factory = model.dataset_factory
  factory.kwargs['train_start'] = train_start
  factory.kwargs['train_end'] = train_end
  factory.kwargs['test_start'] = test_start
  factory.kwargs['test_end'] = test_end
  dataset = factory()

  center = dataset.kwargs['center']
  center = np.float32(center)
  max_val = dataset.kwargs['max_val']
  max_val = np.float32(max_val)
  value_range = max_val * (1. + center)
  min_value = np.float32(0. - center * max_val)

  if 'CIFAR' in str(factory.cls):
    base_eps = 8. / 255.
  elif 'MNIST' in str(factory.cls):
    base_eps = .3
  else:
    raise NotImplementedError(str(factory.cls))

  eps = np.float32(base_eps * value_range)
  clip_min = min_value
  clip_max = max_val

  x_data, y_data = dataset.get_set(which_set)

  nb_classes = dataset.NB_CLASSES

  spsa_max_confidence_recipe(sess, model, x_data, y_data, nb_classes, eps,
                             clip_min, clip_max, nb_iter, report_path,
                             spsa_samples=spsa_samples,
                             spsa_iters=spsa_iters,
                             eval_batch_size=batch_size)