Run all attacks against all defenses and compute results.

def main():
  """Run all attacks against all defenses and compute results.
  """
  args = parse_args()
  attacks_output_dir = os.path.join(args.intermediate_results_dir,
                                    'attacks_output')
  targeted_attacks_output_dir = os.path.join(args.intermediate_results_dir,
                                             'targeted_attacks_output')
  defenses_output_dir = os.path.join(args.intermediate_results_dir,
                                     'defenses_output')
  all_adv_examples_dir = os.path.join(args.intermediate_results_dir,
                                      'all_adv_examples')

  # Load dataset metadata.
  dataset_meta = DatasetMetadata(args.dataset_metadata)

  # Load attacks and defenses.
  attacks = [
      a for a in read_submissions_from_directory(args.attacks_dir,
                                                 args.use_gpu)
      if isinstance(a, Attack)
  ]
  targeted_attacks = [
      a for a in read_submissions_from_directory(args.targeted_attacks_dir,
                                                 args.use_gpu)
      if isinstance(a, Attack)
  ]
  defenses = [
      d for d in read_submissions_from_directory(args.defenses_dir,
                                                 args.use_gpu)
      if isinstance(d, Defense)
  ]
  print('Found attacks: ', [a.name for a in attacks])
  print('Found tageted attacks: ', [a.name for a in targeted_attacks])
  print('Found defenses: ', [d.name for d in defenses])

  # Prepare subdirectories for intermediate results.
  os.mkdir(attacks_output_dir)
  os.mkdir(targeted_attacks_output_dir)
  os.mkdir(defenses_output_dir)
  os.mkdir(all_adv_examples_dir)
  for a in attacks:
    os.mkdir(os.path.join(attacks_output_dir, a.name))
  for a in targeted_attacks:
    os.mkdir(os.path.join(targeted_attacks_output_dir, a.name))
  for d in defenses:
    os.mkdir(os.path.join(defenses_output_dir, d.name))

  # Run all non-targeted attacks.
  attacks_output = AttacksOutput(args.dataset_dir,
                                 attacks_output_dir,
                                 targeted_attacks_output_dir,
                                 all_adv_examples_dir,
                                 args.epsilon)
  for a in attacks:
    a.run(args.dataset_dir,
          os.path.join(attacks_output_dir, a.name),
          args.epsilon)
    attacks_output.clip_and_copy_attack_outputs(a.name, False)

  # Run all targeted attacks.
  dataset_meta.save_target_classes(os.path.join(args.dataset_dir,
                                                'target_class.csv'))
  for a in targeted_attacks:
    a.run(args.dataset_dir,
          os.path.join(targeted_attacks_output_dir, a.name),
          args.epsilon)
    attacks_output.clip_and_copy_attack_outputs(a.name, True)

  # Run all defenses.
  defenses_output = {}
  for d in defenses:
    d.run(all_adv_examples_dir, os.path.join(defenses_output_dir, d.name))
    defenses_output[d.name] = load_defense_output(
        os.path.join(defenses_output_dir, d.name, 'result.csv'))

  # Compute and save scoring.
  compute_and_save_scores_and_ranking(attacks_output, defenses_output,
                                      dataset_meta, args.output_dir,
                                      args.save_all_classification)