import argparse
import os.path as osp
import mmcv
import numpy as np
from mmcv import Config, DictAction
from mmdet.core.evaluation import eval_map
from mmdet.core.visualization import imshow_gt_det_bboxes
from mmdet.datasets import build_dataset, get_loading_pipeline
def bbox_map_eval(det_result, annotation):
"""Evaluate mAP of single image det result.
Args:
det_result (list[list]): [[cls1_det, cls2_det, ...], ...].
The outer list indicates images, and the inner list indicates
per-class detected bboxes.
annotation (dict): Ground truth annotations where keys of
annotations are:
- bboxes: numpy array of shape (n, 4)
- labels: numpy array of shape (n, )
- bboxes_ignore (optional): numpy array of shape (k, 4)
- labels_ignore (optional): numpy array of shape (k, )
Returns:
float: mAP
"""
# use only bbox det result
if isinstance(det_result, tuple):
bbox_det_result = [det_result[0]]
else:
bbox_det_result = [det_result]
# mAP
iou_thrs = np.linspace(
.5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)
mean_aps = []
for thr in iou_thrs:
mean_ap, _ = eval_map(
bbox_det_result, [annotation], iou_thr=thr, logger='silent')
mean_aps.append(mean_ap)
return sum(mean_aps) / len(mean_aps)
class ResultVisualizer(object):
"""Display and save evaluation results.
Args:
show (bool): Whether to show the image. Default: True
wait_time (float): Value of waitKey param. Default: 0.
score_thr (float): Minimum score of bboxes to be shown.
Default: 0
"""
def __init__(self, show=False, wait_time=0, score_thr=0):
self.show = show
self.wait_time = wait_time
self.score_thr = score_thr
def _save_image_gts_results(self, dataset, results, mAPs, out_dir=None):
mmcv.mkdir_or_exist(out_dir)
for mAP_info in mAPs:
index, mAP = mAP_info
data_info = dataset.prepare_train_img(index)
# calc save file path
filename = data_info['filename']
if data_info['img_prefix'] is not None:
filename = osp.join(data_info['img_prefix'], filename)
else:
filename = data_info['filename']
fname, name = osp.splitext(osp.basename(filename))
save_filename = fname + '_' + str(round(mAP, 3)) + name
out_file = osp.join(out_dir, save_filename)
imshow_gt_det_bboxes(
data_info['img'],
data_info,
results[index],
dataset.CLASSES,
show=self.show,
score_thr=self.score_thr,
wait_time=self.wait_time,
out_file=out_file)
def evaluate_and_show(self,
dataset,
results,
topk=20,
show_dir='work_dir',
eval_fn=None):
"""Evaluate and show results.
Args:
dataset (Dataset): A PyTorch dataset.
results (list): Det results from test results pkl file
topk (int): Number of the highest topk and
lowest topk after evaluation index sorting. Default: 20
show_dir (str, optional): The filename to write the image.
Default: 'work_dir'
eval_fn (callable, optional): Eval function, Default: None
"""
assert topk > 0
if (topk * 2) > len(dataset):
topk = len(dataset) // 2
if eval_fn is None:
eval_fn = bbox_map_eval
else:
assert callable(eval_fn)
prog_bar = mmcv.ProgressBar(len(results))
_mAPs = {}
for i, (result, ) in enumerate(zip(results)):
# self.dataset[i] should not call directly
# because there is a risk of mismatch
data_info = dataset.prepare_train_img(i)
mAP = eval_fn(result, data_info['ann_info'])
_mAPs[i] = mAP
prog_bar.update()
# descending select topk image
_mAPs = list(sorted(_mAPs.items(), key=lambda kv: kv[1]))
good_mAPs = _mAPs[-topk:]
bad_mAPs = _mAPs[:topk]
good_dir = osp.abspath(osp.join(show_dir, 'good'))
bad_dir = osp.abspath(osp.join(show_dir, 'bad'))
self._save_image_gts_results(dataset, results, good_mAPs, good_dir)
self._save_image_gts_results(dataset, results, bad_mAPs, bad_dir)
def parse_args():
parser = argparse.ArgumentParser(
description='MMDet eval image prediction result for each')
parser.add_argument('config', help='test config file path')
parser.add_argument(
'prediction_path', help='prediction path where test pkl result')
parser.add_argument(
'show_dir', help='directory where painted images will be saved')
parser.add_argument('--show', action='store_true', help='show results')
parser.add_argument(
'--wait-time',
type=float,
default=0,
help='the interval of show (s), 0 is block')
parser.add_argument(
'--topk',
default=20,
type=int,
help='saved Number of the highest topk '
'and lowest topk after index sorting')
parser.add_argument(
'--show-score-thr',
type=float,
default=0,
help='score threshold (default: 0.)')
parser.add_argument(
'--cfg-options',
nargs='+',
action=DictAction,
help='override some settings in the used config, the key-value pair '
'in xxx=yyy format will be merged into config file. If the value to '
'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
'Note that the quotation marks are necessary and that no white space '
'is allowed.')
args = parser.parse_args()
return args
def main():
args = parse_args()
mmcv.check_file_exist(args.prediction_path)
cfg = Config.fromfile(args.config)
if args.cfg_options is not None:
cfg.merge_from_dict(args.cfg_options)
cfg.data.test.test_mode = True
# import modules from string list.
if cfg.get('custom_imports', None):
from mmcv.utils import import_modules_from_strings
import_modules_from_strings(**cfg['custom_imports'])
cfg.data.test.pop('samples_per_gpu', 0)
cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.pipeline)
dataset = build_dataset(cfg.data.test)
outputs = mmcv.load(args.prediction_path)
result_visualizer = ResultVisualizer(args.show, args.wait_time,
args.show_score_thr)
result_visualizer.evaluate_and_show(
dataset, outputs, topk=args.topk, show_dir=args.show_dir)
if __name__ == '__main__':
main()