Computes results (scores, stats, etc...) of competition evaluation.

    Results are saved into output directory (self.results_dir).
    Also this method saves all intermediate data into output directory as well,
    so it can resume computation if it was interrupted for some reason.
    This is useful because computatin of resuls could take many minutes.

def compute_results(self):
    """Computes results (scores, stats, etc...) of competition evaluation.

    Results are saved into output directory (self.results_dir).
    Also this method saves all intermediate data into output directory as well,
    so it can resume computation if it was interrupted for some reason.
    This is useful because computatin of resuls could take many minutes.
    """
    # read all data
    logging.info('Reading data from datastore')
    dataset_meta = self._read_dataset_metadata()
    self.submissions.init_from_datastore()
    self.dataset_batches.init_from_datastore()
    self.adv_batches.init_from_datastore()
    self.attack_work.read_all_from_datastore()

    if os.path.exists(os.path.join(self.results_dir, 'defense_work.dump')):
      with open(os.path.join(self.results_dir, 'defense_work.dump')) as f:
        self.defense_work.deserialize(f)
    else:
      self.defense_work.read_all_from_datastore()
      with open(os.path.join(self.results_dir, 'defense_work.dump'), 'w') as f:
        self.defense_work.serialize(f)

    if os.path.exists(os.path.join(self.results_dir, 'class_batches.dump')):
      with open(os.path.join(self.results_dir, 'class_batches.dump')) as f:
        self.class_batches.deserialize(f)
    else:
      self.class_batches.init_from_datastore()
      with open(os.path.join(self.results_dir, 'class_batches.dump'), 'w') as f:
        self.class_batches.serialize(f)

    # process data
    logging.info('Processing classification results')
    count_adv_images = self.adv_batches.count_generated_adv_examples()
    intermediate_files = ['acc_matrix.dump', 'error_matrix.dump',
                          'hit_tc_matrix.dump', 'classified_images_count.dump']
    if all([os.path.exists(os.path.join(self.results_dir, fname))
            for fname in intermediate_files]):
      with open(os.path.join(self.results_dir, 'acc_matrix.dump')) as f:
        acc_matrix = pickle.load(f)
      with open(os.path.join(self.results_dir, 'error_matrix.dump')) as f:
        error_matrix = pickle.load(f)
      with open(os.path.join(self.results_dir, 'hit_tc_matrix.dump')) as f:
        hit_tc_matrix = pickle.load(f)
      with open(os.path.join(self.results_dir,
                             'classified_images_count.dump')) as f:
        classified_images_count = pickle.load(f)
    else:
      acc_matrix, error_matrix, hit_tc_matrix, classified_images_count = (
          self.class_batches.compute_classification_results(
              self.adv_batches,
              self.dataset_batches,
              dataset_meta,
              self.defense_work))
      with open(os.path.join(self.results_dir, 'acc_matrix.dump'), 'w') as f:
        pickle.dump(acc_matrix, f)
      with open(os.path.join(self.results_dir, 'error_matrix.dump'), 'w') as f:
        pickle.dump(error_matrix, f)
      with open(os.path.join(self.results_dir, 'hit_tc_matrix.dump'), 'w') as f:
        pickle.dump(hit_tc_matrix, f)
      with open(os.path.join(self.results_dir,
                             'classified_images_count.dump'), 'w') as f:
        pickle.dump(classified_images_count, f)

    # compute attacks and defenses which will be used for scoring
    logging.info('Computing attacks and defenses which are used for scoring')
    expected_num_adv_images = self.dataset_batches.count_num_images()
    attacks_to_use = [k for k, v in iteritems(count_adv_images)
                      if ((v == expected_num_adv_images)
                          and (k not in self.blacklisted_submissions))]

    total_num_adversarial = sum(itervalues(count_adv_images))
    defenses_to_use = [k for k, v in iteritems(classified_images_count)
                       if ((v == total_num_adversarial)
                           and (k not in self.blacklisted_submissions))]

    logging.info('Expected number of adversarial images: %d',
                 expected_num_adv_images)
    logging.info('Number of attacks to use to score defenses: %d',
                 len(attacks_to_use))
    logging.info('Expected number of classification predictions: %d',
                 total_num_adversarial)
    logging.info('Number of defenses to use to score attacks: %d',
                 len(defenses_to_use))

    save_dict_to_file(os.path.join(self.results_dir, 'count_adv_images.csv'),
                      count_adv_images)
    save_dict_to_file(os.path.join(self.results_dir,
                                   'classified_images_count.csv'),
                      classified_images_count)

    # compute scores
    logging.info('Computing scores')
    attack_scores = defaultdict(lambda: 0)
    targeted_attack_scores = defaultdict(lambda: 0)
    defense_scores = defaultdict(lambda: 0)
    for defense_id in acc_matrix.dim0:
      for attack_id in acc_matrix.dim1:
        if attack_id in attacks_to_use:
          defense_scores[defense_id] += acc_matrix[defense_id, attack_id]
        if defense_id in defenses_to_use:
          if attack_id in self.submissions.targeted_attacks:
            targeted_attack_scores[attack_id] += (
                hit_tc_matrix[defense_id, attack_id])
          else:
            attack_scores[attack_id] += error_matrix[defense_id, attack_id]
    # negate results of blacklisted submissions
    for s_id in self.blacklisted_submissions:
      if s_id in defense_scores:
        defense_scores[s_id] = -defense_scores[s_id]
      if s_id in attack_scores:
        attack_scores[s_id] = -attack_scores[s_id]
      if s_id in targeted_attack_scores:
        targeted_attack_scores[s_id] = -targeted_attack_scores[s_id]
    # save results
    logging.info('Saving results')
    all_attack_stats = self.attack_work.compute_work_statistics()
    nontargeted_attack_stats = {k: v for k, v in iteritems(all_attack_stats)
                                if k in self.submissions.attacks}
    targeted_attack_stats = {k: v for k, v in iteritems(all_attack_stats)
                             if k in self.submissions.targeted_attacks}
    defense_stats = self.defense_work.compute_work_statistics()
    self._save_work_results(
        nontargeted_attack_stats, attack_scores, count_adv_images,
        os.path.join(self.results_dir, 'attack_results.csv'))
    self._save_work_results(
        targeted_attack_stats, targeted_attack_scores, count_adv_images,
        os.path.join(self.results_dir, 'targeted_attack_results.csv'))
    self._save_work_results(
        defense_stats, defense_scores,
        classified_images_count,
        os.path.join(self.results_dir, 'defense_results.csv'))

    self._save_sorted_results(
        nontargeted_attack_stats, attack_scores, count_adv_images,
        os.path.join(self.results_dir, 'sorted_attack_results.csv'))
    self._save_sorted_results(
        targeted_attack_stats, targeted_attack_scores, count_adv_images,
        os.path.join(self.results_dir, 'sorted_target_attack_results.csv'))
    self._save_sorted_results(
        defense_stats, defense_scores, classified_images_count,
        os.path.join(self.results_dir, 'sorted_defense_results.csv'))

    defense_id_to_name = {k: self.submissions.get_external_id(k)
                          for k in iterkeys(self.submissions.defenses)}
    attack_id_to_name = {k: self.submissions.get_external_id(k)
                         for k in self.submissions.get_all_attack_ids()}
    acc_matrix.save_to_file(
        os.path.join(self.results_dir, 'accuracy_matrix.csv'),
        remap_dim0=defense_id_to_name, remap_dim1=attack_id_to_name)
    error_matrix.save_to_file(
        os.path.join(self.results_dir, 'error_matrix.csv'),
        remap_dim0=defense_id_to_name, remap_dim1=attack_id_to_name)
    hit_tc_matrix.save_to_file(
        os.path.join(self.results_dir, 'hit_target_class_matrix.csv'),
        remap_dim0=defense_id_to_name, remap_dim1=attack_id_to_name)

    save_dict_to_file(os.path.join(self.results_dir, 'defense_id_to_name.csv'),
                      defense_id_to_name)
    save_dict_to_file(os.path.join(self.results_dir, 'attack_id_to_name.csv'),
                      attack_id_to_name)