r"""Compute action detection performance for the AVA dataset.

Please send any questions about this code to the Google Group ava-dataset-users:
https://groups.google.com/forum/#!forum/ava-dataset-users

Example usage:
python -O get_ava_performance.py \
  -l ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt \
  -g ava_val_v2.1.csv \
  -e ava_val_excluded_timestamps_v2.1.csv \
  -d your_results.csv
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
from collections import defaultdict
import csv
import logging
import pprint
import sys
import time
import numpy as np

from ava import object_detection_evaluation
from ava import standard_fields

import os
import json

ACAM_FOLDER = os.environ['ACAM_DIR']
AVA_FOLDER = ACAM_FOLDER + '/data/AVA' 
RESULTS_FOLDER = AVA_FOLDER + '/result_mAPs'

def print_time(message, start):
  logging.info("==> %g seconds to %s", time.time() - start, message)


def make_image_key(video_id, timestamp):
  """Returns a unique identifier for a video id & timestamp."""
  return "%s,%04d" % (video_id, int(timestamp))


def read_csv(csv_file, class_whitelist=None):
  """Loads boxes and class labels from a CSV file in the AVA format.

  CSV file format described at https://research.google.com/ava/download.html.

  Args:
    csv_file: A file object.
    class_whitelist: If provided, boxes corresponding to (integer) class labels
      not in this set are skipped.

  Returns:
    boxes: A dictionary mapping each unique image key (string) to a list of
      boxes, given as coordinates [y1, x1, y2, x2].
    labels: A dictionary mapping each unique image key (string) to a list of
      integer class lables, matching the corresponding box in `boxes`.
    scores: A dictionary mapping each unique image key (string) to a list of
      score values lables, matching the corresponding label in `labels`. If
      scores are not provided in the csv, then they will default to 1.0.
  """
  start = time.time()
  boxes = defaultdict(list)
  labels = defaultdict(list)
  scores = defaultdict(list)
  reader = csv.reader(csv_file)
  for row in reader:
    assert len(row) in [7, 8], "Wrong number of columns: " + row
    image_key = make_image_key(row[0], row[1])
    x1, y1, x2, y2 = [float(n) for n in row[2:6]]
    action_id = int(row[6])
    if class_whitelist and action_id not in class_whitelist:
      continue
    score = 1.0
    if len(row) == 8:
      score = float(row[7])
    boxes[image_key].append([y1, x1, y2, x2])
    labels[image_key].append(action_id)
    scores[image_key].append(score)
  print_time("read file " + csv_file.name, start)
  return boxes, labels, scores


def read_exclusions(exclusions_file):
  """Reads a CSV file of excluded timestamps.

  Args:
    exclusions_file: A file object containing a csv of video-id,timestamp.

  Returns:
    A set of strings containing excluded image keys, e.g. "<KEY>",
    or an empty set if exclusions file is None.
  """
  excluded = set()
  if exclusions_file:
    reader = csv.reader(exclusions_file)
    for row in reader:
      assert len(row) == 2, "Expected only 2 columns, got: " + row
      excluded.add(make_image_key(row[0], row[1]))
  return excluded


def read_labelmap(labelmap_file):
  """Reads a labelmap without the dependency on protocol buffers.

  Args:
    labelmap_file: A file object containing a label map protocol buffer.

  Returns:
    labelmap: The label map in the form used by the object_detection_evaluation
      module - a list of {"id": integer, "name": classname } dicts.
    class_ids: A set containing all of the valid class id integers.
  """
  labelmap = []
  class_ids = set()
  name = ""
  class_id = ""
  for line in labelmap_file:
    if line.startswith("  name:"):
      name = line.split('"')[1]
    elif line.startswith("  id:") or line.startswith("  label_id:"):
      class_id = int(line.strip().split(" ")[-1])
      labelmap.append({"id": class_id, "name": name})
      class_ids.add(class_id)
  return labelmap, class_ids


def run_evaluation(labelmap, groundtruth, detections, exclusions):
  """Runs evaluations given input files.

  Args:
    labelmap: file object containing map of labels to consider, in pbtxt format
    groundtruth: file object
    detections: file object
    exclusions: file object or None.
  """
  categories, class_whitelist = read_labelmap(labelmap)
  logging.info("CATEGORIES (%d):\n%s", len(categories),
               pprint.pformat(categories, indent=2))
  excluded_keys = read_exclusions(exclusions)

  pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator(
      categories)

  # Reads the ground truth data.
  boxes, labels, _ = read_csv(groundtruth, class_whitelist)
  start = time.time()
  for image_key in boxes:
    if image_key in excluded_keys:
      logging.info(("Found excluded timestamp in ground truth: %s. "
                    "It will be ignored."), image_key)
      continue
    pascal_evaluator.add_single_ground_truth_image_info(
        image_key, {
            standard_fields.InputDataFields.groundtruth_boxes:
                np.array(boxes[image_key], dtype=float),
            standard_fields.InputDataFields.groundtruth_classes:
                np.array(labels[image_key], dtype=int),
            standard_fields.InputDataFields.groundtruth_difficult:
                np.zeros(len(boxes[image_key]), dtype=bool)
        })
  print_time("convert groundtruth", start)

  # Reads detections data.
  boxes, labels, scores = read_csv(detections, class_whitelist)
  start = time.time()
  for image_key in boxes:
    if image_key in excluded_keys:
      logging.info(("Found excluded timestamp in detections: %s. "
                    "It will be ignored."), image_key)
      continue
    pascal_evaluator.add_single_detected_image_info(
        image_key, {
            standard_fields.DetectionResultFields.detection_boxes:
                np.array(boxes[image_key], dtype=float),
            standard_fields.DetectionResultFields.detection_classes:
                np.array(labels[image_key], dtype=int),
            standard_fields.DetectionResultFields.detection_scores:
                np.array(scores[image_key], dtype=float)
        })
  print_time("convert detections", start)

  start = time.time()
  metrics = pascal_evaluator.evaluate()
  print_time("run_evaluator", start)
  # pprint.pprint(metrics, indent=2)

  return metrics

def combine_res_with_size(result_list):
  data_dir = AVA_FOLDER + '/data/'
  with open(data_dir + 'action_lists_train.json') as fp:
    tr_lists = json.load(fp)
  with open(data_dir + 'action_lists_val.json') as fp:
    val_lists = json.load(fp)
  with open(data_dir + 'label_conversions.json') as fp:
    label_conv = json.load(fp)

  # scale mAPs with *100 for readability
  res_dict = {r[0].split('/AP@0.5IOU/')[-1]:100*r[1] for r in result_list}
  # {u'PascalBoxes_Precision/mAP@0.5IOU': 0.16426229802003,
  # u'answer phone': 0.4416065942965194,
  # u'bend/bow (at the waist)': 0.29166019646294344,
  # u'carry/hold (an object)': 0.4682357576724985,

  cls_nos = label_conv['ann2train'].keys()

  tr_sizes = {label_conv['ann2train'][cls_no_str]['class_str']: len(tr_lists[cls_no_str]) for cls_no_str in cls_nos}
  val_sizes = {label_conv['ann2train'][cls_no_str]['class_str']: len(val_lists[cls_no_str]) for cls_no_str in cls_nos}
  # {u'answer phone': 3251,
  #  u'bend/bow (at the waist)': 8148,

  cls_strs = tr_sizes.keys()

  combined = [(cls_str, tr_sizes[cls_str], val_sizes[cls_str], res_dict[cls_str]) for cls_str in cls_strs]
  mAP = res_dict['PascalBoxes_Precision/mAP@0.5IOU']

  # sort by training size,ie index 1
  combined.sort(key=lambda x: x[1])

  print('\n')
  print("{:<40}{:<15}{:<15}{}".format('Action Class', 'Train Size', 'Val Size', 'Class AP'))
  for term in combined:
    # cls_str, tr_size, val_size, cls_ap = term
    print("{:<40}{:<15}{:<15}{:.2f}".format(*term))

  # 14: walk (person movement), 15: answer_phone (object manipulation), ..., 63: write (object man), 64: fight/hit person (person interaction)
  movement_APs = []
  object_APs = []
  inter_APs = []
  for cls_no_str in cls_nos:
    cls_str = label_conv['ann2train'][cls_no_str]['class_str']
    class_AP = res_dict[cls_str]
    if int(cls_no_str) <= 14:
        movement_APs.append(class_AP)
    elif int(cls_no_str) <= 63:
        object_APs.append(class_AP)
    elif int(cls_no_str) <= 80:
        inter_APs.append(class_AP)
    else:
        print("Wrong class id error")

  print('\n')
  print('\t\tMovement mAP:    %.4f' % np.mean(movement_APs))
  print('\t\tObject mAP:      %.4f' % np.mean(object_APs))
  print('\t\tInteraction mAP: %.4f' % np.mean(inter_APs))

  print('\n\t\tFinal mAP: %.4f\n' % mAP)


def parse_arguments():
  """Parses command-line flags.

  Returns:
    args: a named tuple containing three file objects args.labelmap,
    args.groundtruth, and args.detections.
  """
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "-l",
      "--labelmap",
      help="Filename of label map",
      type=argparse.FileType("r"),
      default="ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt")
  parser.add_argument(
      "-g",
      "--groundtruth",
      help="CSV file containing ground truth.",
      type=argparse.FileType("r"),
      required=True)
  parser.add_argument(
      "-d",
      "--detections",
      help="CSV file containing inferred action detections.",
      type=argparse.FileType("r"),
      required=True)
  parser.add_argument(
      "-e",
      "--exclusions",
      help=("Optional CSV file containing videoid,timestamp pairs to exclude "
            "from evaluation."),
      type=argparse.FileType("r"),
      required=False)
  return parser.parse_args()


def main():
  logging.basicConfig(level=logging.INFO)
  args = parse_arguments()
  metrics = run_evaluation(**vars(args))

  # make metrics more human readable
  result_list = metrics.items()
  result_list.sort(key=lambda x:x[0])

  combine_res_with_size(result_list)

  detection_filename_ext = os.path.basename(vars(args)['detections'].name)
  detection_name = detection_filename_ext.split('.')[0]

  output_file_path = '%s/%s.json' % (RESULTS_FOLDER, detection_name)
  with open(output_file_path, 'w') as fp:
      json.dump(result_list, fp, indent=2)
  print('Results are saved to %s' % output_file_path)


if __name__ == "__main__":
  main()

r"""Compute action detection performance for the AVA dataset.

Please send any questions about this code to the Google Group ava-dataset-users:
https://groups.google.com/forum/#!forum/ava-dataset-users

Example usage:
python -O get_ava_performance.py \
  -l ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt \
  -g ava_val_v2.1.csv \
  -e ava_val_excluded_timestamps_v2.1.csv \
  -d your_results.csv
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
from collections import defaultdict
import csv
import logging
import pprint
import sys
import time
import numpy as np

from ava import object_detection_evaluation
from ava import standard_fields

import os
import json

ACAM_FOLDER = os.environ['ACAM_DIR']
AVA_FOLDER = ACAM_FOLDER + '/data/AVA' 
RESULTS_FOLDER = AVA_FOLDER + '/result_mAPs'

def print_time(message, start):
  logging.info("==> %g seconds to %s", time.time() - start, message)


def make_image_key(video_id, timestamp):
  """Returns a unique identifier for a video id & timestamp."""
  return "%s,%04d" % (video_id, int(timestamp))


def read_csv(csv_file, class_whitelist=None):
  """Loads boxes and class labels from a CSV file in the AVA format.

  CSV file format described at https://research.google.com/ava/download.html.

  Args:
    csv_file: A file object.
    class_whitelist: If provided, boxes corresponding to (integer) class labels
      not in this set are skipped.

  Returns:
    boxes: A dictionary mapping each unique image key (string) to a list of
      boxes, given as coordinates [y1, x1, y2, x2].
    labels: A dictionary mapping each unique image key (string) to a list of
      integer class lables, matching the corresponding box in `boxes`.
    scores: A dictionary mapping each unique image key (string) to a list of
      score values lables, matching the corresponding label in `labels`. If
      scores are not provided in the csv, then they will default to 1.0.
  """
  start = time.time()
  boxes = defaultdict(list)
  labels = defaultdict(list)
  scores = defaultdict(list)
  reader = csv.reader(csv_file)
  for row in reader:
    assert len(row) in [7, 8], "Wrong number of columns: " + row
    image_key = make_image_key(row[0], row[1])
    x1, y1, x2, y2 = [float(n) for n in row[2:6]]
    action_id = int(row[6])
    if class_whitelist and action_id not in class_whitelist:
      continue
    score = 1.0
    if len(row) == 8:
      score = float(row[7])
    boxes[image_key].append([y1, x1, y2, x2])
    labels[image_key].append(action_id)
    scores[image_key].append(score)
  print_time("read file " + csv_file.name, start)
  return boxes, labels, scores


def read_exclusions(exclusions_file):
  """Reads a CSV file of excluded timestamps.

  Args:
    exclusions_file: A file object containing a csv of video-id,timestamp.

  Returns:
    A set of strings containing excluded image keys, e.g. "<KEY>",
    or an empty set if exclusions file is None.
  """
  excluded = set()
  if exclusions_file:
    reader = csv.reader(exclusions_file)
    for row in reader:
      assert len(row) == 2, "Expected only 2 columns, got: " + row
      excluded.add(make_image_key(row[0], row[1]))
  return excluded


def read_labelmap(labelmap_file):
  """Reads a labelmap without the dependency on protocol buffers.

  Args:
    labelmap_file: A file object containing a label map protocol buffer.

  Returns:
    labelmap: The label map in the form used by the object_detection_evaluation
      module - a list of {"id": integer, "name": classname } dicts.
    class_ids: A set containing all of the valid class id integers.
  """
  labelmap = []
  class_ids = set()
  name = ""
  class_id = ""
  for line in labelmap_file:
    if line.startswith("  name:"):
      name = line.split('"')[1]
    elif line.startswith("  id:") or line.startswith("  label_id:"):
      class_id = int(line.strip().split(" ")[-1])
      labelmap.append({"id": class_id, "name": name})
      class_ids.add(class_id)
  return labelmap, class_ids


def run_evaluation(labelmap, groundtruth, detections, exclusions):
  """Runs evaluations given input files.

  Args:
    labelmap: file object containing map of labels to consider, in pbtxt format
    groundtruth: file object
    detections: file object
    exclusions: file object or None.
  """
  categories, class_whitelist = read_labelmap(labelmap)
  logging.info("CATEGORIES (%d):\n%s", len(categories),
               pprint.pformat(categories, indent=2))
  excluded_keys = read_exclusions(exclusions)

  pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator(
      categories)

  # Reads the ground truth data.
  boxes, labels, _ = read_csv(groundtruth, class_whitelist)
  start = time.time()
  for image_key in boxes:
    if image_key in excluded_keys:
      logging.info(("Found excluded timestamp in ground truth: %s. "
                    "It will be ignored."), image_key)
      continue
    pascal_evaluator.add_single_ground_truth_image_info(
        image_key, {
            standard_fields.InputDataFields.groundtruth_boxes:
                np.array(boxes[image_key], dtype=float),
            standard_fields.InputDataFields.groundtruth_classes:
                np.array(labels[image_key], dtype=int),
            standard_fields.InputDataFields.groundtruth_difficult:
                np.zeros(len(boxes[image_key]), dtype=bool)
        })
  print_time("convert groundtruth", start)

  # Reads detections data.
  boxes, labels, scores = read_csv(detections, class_whitelist)
  start = time.time()
  for image_key in boxes:
    if image_key in excluded_keys:
      logging.info(("Found excluded timestamp in detections: %s. "
                    "It will be ignored."), image_key)
      continue
    pascal_evaluator.add_single_detected_image_info(
        image_key, {
            standard_fields.DetectionResultFields.detection_boxes:
                np.array(boxes[image_key], dtype=float),
            standard_fields.DetectionResultFields.detection_classes:
                np.array(labels[image_key], dtype=int),
            standard_fields.DetectionResultFields.detection_scores:
                np.array(scores[image_key], dtype=float)
        })
  print_time("convert detections", start)

  start = time.time()
  metrics = pascal_evaluator.evaluate()
  print_time("run_evaluator", start)
  # pprint.pprint(metrics, indent=2)

  return metrics

def combine_res_with_size(result_list):
  data_dir = AVA_FOLDER + '/data/'
  with open(data_dir + 'action_lists_train.json') as fp:
    tr_lists = json.load(fp)
  with open(data_dir + 'action_lists_val.json') as fp:
    val_lists = json.load(fp)
  with open(data_dir + 'label_conversions.json') as fp:
    label_conv = json.load(fp)

  # scale mAPs with *100 for readability
  res_dict = {r[0].split('/AP@0.5IOU/')[-1]:100*r[1] for r in result_list}
  # {u'PascalBoxes_Precision/mAP@0.5IOU': 0.16426229802003,
  # u'answer phone': 0.4416065942965194,
  # u'bend/bow (at the waist)': 0.29166019646294344,
  # u'carry/hold (an object)': 0.4682357576724985,

  cls_nos = label_conv['ann2train'].keys()

  tr_sizes = {label_conv['ann2train'][cls_no_str]['class_str']: len(tr_lists[cls_no_str]) for cls_no_str in cls_nos}
  val_sizes = {label_conv['ann2train'][cls_no_str]['class_str']: len(val_lists[cls_no_str]) for cls_no_str in cls_nos}
  # {u'answer phone': 3251,
  #  u'bend/bow (at the waist)': 8148,

  cls_strs = tr_sizes.keys()

  combined = [(cls_str, tr_sizes[cls_str], val_sizes[cls_str], res_dict[cls_str]) for cls_str in cls_strs]
  mAP = res_dict['PascalBoxes_Precision/mAP@0.5IOU']

  # sort by training size,ie index 1
  combined.sort(key=lambda x: x[1])

  print('\n')
  print("{:<40}{:<15}{:<15}{}".format('Action Class', 'Train Size', 'Val Size', 'Class AP'))
  for term in combined:
    # cls_str, tr_size, val_size, cls_ap = term
    print("{:<40}{:<15}{:<15}{:.2f}".format(*term))

  # 14: walk (person movement), 15: answer_phone (object manipulation), ..., 63: write (object man), 64: fight/hit person (person interaction)
  movement_APs = []
  object_APs = []
  inter_APs = []
  for cls_no_str in cls_nos:
    cls_str = label_conv['ann2train'][cls_no_str]['class_str']
    class_AP = res_dict[cls_str]
    if int(cls_no_str) <= 14:
        movement_APs.append(class_AP)
    elif int(cls_no_str) <= 63:
        object_APs.append(class_AP)
    elif int(cls_no_str) <= 80:
        inter_APs.append(class_AP)
    else:
        print("Wrong class id error")

  print('\n')
  print('\t\tMovement mAP:    %.4f' % np.mean(movement_APs))
  print('\t\tObject mAP:      %.4f' % np.mean(object_APs))
  print('\t\tInteraction mAP: %.4f' % np.mean(inter_APs))

  print('\n\t\tFinal mAP: %.4f\n' % mAP)


def parse_arguments():
  """Parses command-line flags.

  Returns:
    args: a named tuple containing three file objects args.labelmap,
    args.groundtruth, and args.detections.
  """
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "-l",
      "--labelmap",
      help="Filename of label map",
      type=argparse.FileType("r"),
      default="ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt")
  parser.add_argument(
      "-g",
      "--groundtruth",
      help="CSV file containing ground truth.",
      type=argparse.FileType("r"),
      required=True)
  parser.add_argument(
      "-d",
      "--detections",
      help="CSV file containing inferred action detections.",
      type=argparse.FileType("r"),
      required=True)
  parser.add_argument(
      "-e",
      "--exclusions",
      help=("Optional CSV file containing videoid,timestamp pairs to exclude "
            "from evaluation."),
      type=argparse.FileType("r"),
      required=False)
  return parser.parse_args()


def main():
  logging.basicConfig(level=logging.INFO)
  args = parse_arguments()
  metrics = run_evaluation(**vars(args))

  # make metrics more human readable
  result_list = metrics.items()
  result_list.sort(key=lambda x:x[0])

  combine_res_with_size(result_list)

  detection_filename_ext = os.path.basename(vars(args)['detections'].name)
  detection_name = detection_filename_ext.split('.')[0]

  output_file_path = '%s/%s.json' % (RESULTS_FOLDER, detection_name)
  with open(output_file_path, 'w') as fp:
      json.dump(result_list, fp, indent=2)
  print('Results are saved to %s' % output_file_path)


if __name__ == "__main__":
  main()

Compute action detection performance for the AVA dataset.  Please send any questions about this code to the Google Group ava-dataset-users: https://groups.google.com/forum/#!forum/ava-dataset-users  Example usage: python -O get_ava_performance.py \   -l ava/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt \   -g ava_val_v2.1.csv \   -e ava_val_excluded_timestamps_v2.1.csv \   -d your_results.csv Returns a unique identifier for a video id & timestamp. Loads boxes and class labels from a CSV file in the AVA format.    CSV file format described at https://research.google.com/ava/download.html.    Args:     csv_file: A file object.     class_whitelist: If provided, boxes corresponding to (integer) class labels       not in this set are skipped.    Returns:     boxes: A dictionary mapping each unique image key (string) to a list of       boxes, given as coordinates [y1, x1, y2, x2].     labels: A dictionary mapping each unique image key (string) to a list of       integer class lables, matching the corresponding box in `boxes`.     scores: A dictionary mapping each unique image key (string) to a list of       score values lables, matching the corresponding label in `labels`. If       scores are not provided in the csv, then they will default to 1.0. Reads a CSV file of excluded timestamps.    Args:     exclusions_file: A file object containing a csv of video-id,timestamp.    Returns:     A set of strings containing excluded image keys, e.g. "<KEY>",     or an empty set if exclusions file is None. Reads a labelmap without the dependency on protocol buffers.    Args:     labelmap_file: A file object containing a label map protocol buffer.    Returns:     labelmap: The label map in the form used by the object_detection_evaluation       module - a list of {"id": integer, "name": classname } dicts.     class_ids: A set containing all of the valid class id integers. Runs evaluations given input files.    Args:     labelmap: file object containing map of labels to consider, in pbtxt format     groundtruth: file object     detections: file object     exclusions: file object or None. # Reads the ground truth data. # Reads detections data. # pprint.pprint(metrics, indent=2) # scale mAPs with *100 for readability # {u'PascalBoxes_Precision/mAP@0.5IOU': 0.16426229802003, # u'answer phone': 0.4416065942965194, # u'bend/bow (at the waist)': 0.29166019646294344, # u'carry/hold (an object)': 0.4682357576724985, # {u'answer phone': 3251, #  u'bend/bow (at the waist)': 8148, # sort by training size,ie index 1 # cls_str, tr_size, val_size, cls_ap = term # 14: walk (person movement), 15: answer_phone (object manipulation), ..., 63: write (object man), 64: fight/hit person (person interaction) Parses command-line flags.    Returns:     args: a named tuple containing three file objects args.labelmap,     args.groundtruth, and args.detections. # make metrics more human readable