detectron2/tests/data/test_coco_evaluation.py

# Copyright (c) Facebook, Inc. and its affiliates.
import contextlib
import copy
import io
import json
import numpy as np
import os
import tempfile
import unittest
import torch
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval

from detectron2.data import DatasetCatalog
from detectron2.evaluation import COCOEvaluator
from detectron2.evaluation.fast_eval_api import COCOeval_opt
from detectron2.structures import Boxes, Instances


class TestCOCOeval(unittest.TestCase):
    def test_fast_eval(self):
        # A small set of images/categories from COCO val
        # fmt: off
        detections = [{"image_id": 139, "category_id": 1, "bbox": [417.3332824707031, 159.27003479003906, 47.66064453125, 143.00193786621094], "score": 0.9949821829795837, "segmentation": {"size": [426, 640], "counts": "Tc`52W=3N0N4aNN^E7]:4XE1g:8kDMT;U100000001O1gE[Nk8h1dFiNY9Z1aFkN]9g2J3NdN`FlN`9S1cFRN07]9g1bFoM6;X9c1cFoM=8R9g1bFQN>3U9Y30O01OO1O001N2O1N1O4L4L5UNoE3V:CVF6Q:@YF9l9@ZF<k9[O`F=];HYnX2"}}, {"image_id": 139, "category_id": 1, "bbox": [383.5909118652344, 172.0777587890625, 17.959075927734375, 36.94813537597656], "score": 0.7685421705245972, "segmentation": {"size": [426, 640], "counts": "lZP5m0Z<300O100O100000001O00]OlC0T<OnCOT<OnCNX<JnC2bQT3"}}, {"image_id": 139, "category_id": 1, "bbox": [457.8359069824219, 158.88027954101562, 9.89764404296875, 8.771820068359375], "score": 0.07092753797769547, "segmentation": {"size": [426, 640], "counts": "bSo54T=2N2O1001O006ImiW2"}}] # noqa
        gt_annotations = {"categories": [{"supercategory": "person", "id": 1, "name": "person"}, {"supercategory": "furniture", "id": 65, "name": "bed"}], "images": [{"license": 4, "file_name": "000000000285.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000285.jpg", "height": 640, "width": 586, "date_captured": "2013-11-18 13:09:47", "flickr_url": "http://farm8.staticflickr.com/7434/9138147604_c6225224b8_z.jpg", "id": 285}, {"license": 2, "file_name": "000000000139.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000139.jpg", "height": 426, "width": 640, "date_captured": "2013-11-21 01:34:01", "flickr_url": "http://farm9.staticflickr.com/8035/8024364858_9c41dc1666_z.jpg", "id": 139}], "annotations": [{"segmentation": [[428.19, 219.47, 430.94, 209.57, 430.39, 210.12, 421.32, 216.17, 412.8, 217.27, 413.9, 214.24, 422.42, 211.22, 429.29, 201.6, 430.67, 181.8, 430.12, 175.2, 427.09, 168.06, 426.27, 164.21, 430.94, 159.26, 440.29, 157.61, 446.06, 163.93, 448.53, 168.06, 448.53, 173.01, 449.08, 174.93, 454.03, 185.1, 455.41, 188.4, 458.43, 195.0, 460.08, 210.94, 462.28, 226.61, 460.91, 233.76, 454.31, 234.04, 460.08, 256.85, 462.56, 268.13, 465.58, 290.67, 465.85, 293.14, 463.38, 295.62, 452.66, 295.34, 448.26, 294.52, 443.59, 282.7, 446.06, 235.14, 446.34, 230.19, 438.09, 232.39, 438.09, 221.67, 434.24, 221.12, 427.09, 219.74]], "area": 2913.1103999999987, "iscrowd": 0, "image_id": 139, "bbox": [412.8, 157.61, 53.05, 138.01], "category_id": 1, "id": 230831}, {"segmentation": [[384.98, 206.58, 384.43, 199.98, 385.25, 193.66, 385.25, 190.08, 387.18, 185.13, 387.18, 182.93, 386.08, 181.01, 385.25, 178.81, 385.25, 175.79, 388.0, 172.76, 394.88, 172.21, 398.72, 173.31, 399.27, 176.06, 399.55, 183.48, 397.9, 185.68, 395.15, 188.98, 396.8, 193.38, 398.45, 194.48, 399.0, 205.75, 395.43, 207.95, 388.83, 206.03]], "area": 435.1449499999997, "iscrowd": 0, "image_id": 139, "bbox": [384.43, 172.21, 15.12, 35.74], "category_id": 1, "id": 233201}]} # noqa
        # fmt: on

        # Test a small dataset for typical COCO format
        experiments = {"full": (detections, gt_annotations, {})}

        # Test what happens if the list of detections or ground truth annotations is empty
        experiments["empty_dt"] = ([], gt_annotations, {})
        gt = copy.deepcopy(gt_annotations)
        gt["annotations"] = []
        experiments["empty_gt"] = (detections, gt, {})

        # Test changing parameter settings
        experiments["no_categories"] = (detections, gt_annotations, {"useCats": 0})
        experiments["no_ious"] = (detections, gt_annotations, {"iouThrs": []})
        experiments["no_rec_thrs"] = (detections, gt_annotations, {"recThrs": []})
        experiments["no_max_dets"] = (detections, gt_annotations, {"maxDets": []})
        experiments["one_max_det"] = (detections, gt_annotations, {"maxDets": [1]})
        experiments["no_area"] = (detections, gt_annotations, {"areaRng": [], "areaRngLbl": []})

        # Test what happens if one omits different fields from the annotation structure
        annotation_fields = [
            "id",
            "image_id",
            "category_id",
            "score",
            "area",
            "iscrowd",
            "ignore",
            "bbox",
            "segmentation",
        ]
        for a in annotation_fields:
            gt = copy.deepcopy(gt_annotations)
            for g in gt["annotations"]:
                if a in g:
                    del g[a]
            dt = copy.deepcopy(detections)
            for d in dt:
                if a in d:
                    del d[a]
            experiments["omit_gt_" + a] = (detections, gt, {})
            experiments["omit_dt_" + a] = (dt, gt_annotations, {})

        # Compare precision/recall for original COCO PythonAPI to custom optimized one
        for name, (dt, gt, params) in experiments.items():
            # Dump to json.
            try:
                with tempfile.TemporaryDirectory() as tmpdir:
                    json_file_name = os.path.join(tmpdir, "gt_" + name + ".json")
                    with open(json_file_name, "w") as f:
                        json.dump(gt, f)
                    with contextlib.redirect_stdout(io.StringIO()):
                        coco_api = COCO(json_file_name)
            except Exception:
                pass

            for iou_type in ["bbox", "segm", "keypoints"]:
                # Run original COCOeval PythonAPI
                api_exception = None
                try:
                    with contextlib.redirect_stdout(io.StringIO()):
                        coco_dt = coco_api.loadRes(dt)
                        coco_eval = COCOeval(coco_api, coco_dt, iou_type)
                        for p, v in params.items():
                            setattr(coco_eval.params, p, v)
                        coco_eval.evaluate()
                        coco_eval.accumulate()
                        coco_eval.summarize()
                except Exception as ex:
                    api_exception = ex

                # Run optimized COCOeval_opt API
                opt_exception = None
                try:
                    with contextlib.redirect_stdout(io.StringIO()):
                        coco_dt = coco_api.loadRes(dt)
                        coco_eval_opt = COCOeval_opt(coco_api, coco_dt, iou_type)
                        for p, v in params.items():
                            setattr(coco_eval_opt.params, p, v)
                        coco_eval_opt.evaluate()
                        coco_eval_opt.accumulate()
                        coco_eval_opt.summarize()
                except Exception as ex:
                    opt_exception = ex

                if api_exception is not None and opt_exception is not None:
                    # Original API and optimized API should throw the same exception if annotation
                    # format is bad
                    api_error = "" if api_exception is None else type(api_exception).__name__
                    opt_error = "" if opt_exception is None else type(opt_exception).__name__
                    msg = "%s: comparing COCO APIs, '%s' != '%s'" % (name, api_error, opt_error)
                    self.assertTrue(api_error == opt_error, msg=msg)
                else:
                    # Original API and optimized API should produce the same precision/recalls
                    for k in ["precision", "recall"]:
                        diff = np.abs(coco_eval.eval[k] - coco_eval_opt.eval[k])
                        abs_diff = np.max(diff) if diff.size > 0 else 0.0
                        msg = "%s: comparing COCO APIs, %s differs by %f" % (name, k, abs_diff)
                        self.assertTrue(abs_diff < 1e-4, msg=msg)

    def test_unknown_category(self):
        dataset = "coco_2017_val_100"
        evaluator = COCOEvaluator(dataset)
        evaluator.reset()
        inputs = DatasetCatalog.get(dataset)[:2]
        pred = Instances((100, 100))
        pred.pred_boxes = Boxes(torch.rand(2, 4))
        pred.scores = torch.rand(2)
        pred.pred_classes = torch.tensor([10, 80])
        output = {"instances": pred}
        evaluator.process(inputs, [output, output])
        with self.assertRaises(AssertionError):
            evaluator.evaluate()