|
|
|
import argparse |
|
import os |
|
import os.path as osp |
|
|
|
from mmengine.config import Config, DictAction |
|
from mmengine.dist import init_dist |
|
from mmengine.fileio import dump |
|
from mmengine.utils import mkdir_or_exist |
|
from terminaltables import GithubFlavoredMarkdownTable |
|
|
|
from tools.analysis_tools.benchmark import repeat_measure_inference_speed |
|
|
|
|
|
def parse_args(): |
|
parser = argparse.ArgumentParser( |
|
description='MMDet benchmark a model of FPS') |
|
parser.add_argument('config', help='test config file path') |
|
parser.add_argument('checkpoint_root', help='Checkpoint file root path') |
|
parser.add_argument( |
|
'--round-num', |
|
type=int, |
|
default=1, |
|
help='round a number to a given precision in decimal digits') |
|
parser.add_argument( |
|
'--repeat-num', |
|
type=int, |
|
default=1, |
|
help='number of repeat times of measurement for averaging the results') |
|
parser.add_argument( |
|
'--out', type=str, help='output path of gathered fps to be stored') |
|
parser.add_argument( |
|
'--max-iter', type=int, default=2000, help='num of max iter') |
|
parser.add_argument( |
|
'--log-interval', type=int, default=50, help='interval of logging') |
|
parser.add_argument( |
|
'--fuse-conv-bn', |
|
action='store_true', |
|
help='Whether to fuse conv and bn, this will slightly increase' |
|
'the inference speed') |
|
parser.add_argument( |
|
'--cfg-options', |
|
nargs='+', |
|
action=DictAction, |
|
help='override some settings in the used config, the key-value pair ' |
|
'in xxx=yyy format will be merged into config file. If the value to ' |
|
'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' |
|
'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' |
|
'Note that the quotation marks are necessary and that no white space ' |
|
'is allowed.') |
|
parser.add_argument( |
|
'--launcher', |
|
choices=['none', 'pytorch', 'slurm', 'mpi'], |
|
default='none', |
|
help='job launcher') |
|
parser.add_argument('--local_rank', type=int, default=0) |
|
args = parser.parse_args() |
|
if 'LOCAL_RANK' not in os.environ: |
|
os.environ['LOCAL_RANK'] = str(args.local_rank) |
|
return args |
|
|
|
|
|
def results2markdown(result_dict): |
|
table_data = [] |
|
is_multiple_results = False |
|
for cfg_name, value in result_dict.items(): |
|
name = cfg_name.replace('configs/', '') |
|
fps = value['fps'] |
|
ms_times_pre_image = value['ms_times_pre_image'] |
|
if isinstance(fps, list): |
|
is_multiple_results = True |
|
mean_fps = value['mean_fps'] |
|
mean_times_pre_image = value['mean_times_pre_image'] |
|
fps_str = ','.join([str(s) for s in fps]) |
|
ms_times_pre_image_str = ','.join( |
|
[str(s) for s in ms_times_pre_image]) |
|
table_data.append([ |
|
name, fps_str, mean_fps, ms_times_pre_image_str, |
|
mean_times_pre_image |
|
]) |
|
else: |
|
table_data.append([name, fps, ms_times_pre_image]) |
|
|
|
if is_multiple_results: |
|
table_data.insert(0, [ |
|
'model', 'fps', 'mean_fps', 'times_pre_image(ms)', |
|
'mean_times_pre_image(ms)' |
|
]) |
|
|
|
else: |
|
table_data.insert(0, ['model', 'fps', 'times_pre_image(ms)']) |
|
table = GithubFlavoredMarkdownTable(table_data) |
|
print(table.table, flush=True) |
|
|
|
|
|
if __name__ == '__main__': |
|
args = parse_args() |
|
assert args.round_num >= 0 |
|
assert args.repeat_num >= 1 |
|
|
|
config = Config.fromfile(args.config) |
|
|
|
if args.launcher == 'none': |
|
raise NotImplementedError('Only supports distributed mode') |
|
else: |
|
init_dist(args.launcher) |
|
|
|
result_dict = {} |
|
for model_key in config: |
|
model_infos = config[model_key] |
|
if not isinstance(model_infos, list): |
|
model_infos = [model_infos] |
|
for model_info in model_infos: |
|
record_metrics = model_info['metric'] |
|
cfg_path = model_info['config'].strip() |
|
cfg = Config.fromfile(cfg_path) |
|
checkpoint = osp.join(args.checkpoint_root, |
|
model_info['checkpoint'].strip()) |
|
try: |
|
fps = repeat_measure_inference_speed(cfg, checkpoint, |
|
args.max_iter, |
|
args.log_interval, |
|
args.fuse_conv_bn, |
|
args.repeat_num) |
|
if args.repeat_num > 1: |
|
fps_list = [round(fps_, args.round_num) for fps_ in fps] |
|
times_pre_image_list = [ |
|
round(1000 / fps_, args.round_num) for fps_ in fps |
|
] |
|
mean_fps = round( |
|
sum(fps_list) / len(fps_list), args.round_num) |
|
mean_times_pre_image = round( |
|
sum(times_pre_image_list) / len(times_pre_image_list), |
|
args.round_num) |
|
print( |
|
f'{cfg_path} ' |
|
f'Overall fps: {fps_list}[{mean_fps}] img / s, ' |
|
f'times per image: ' |
|
f'{times_pre_image_list}[{mean_times_pre_image}] ' |
|
f'ms / img', |
|
flush=True) |
|
result_dict[cfg_path] = dict( |
|
fps=fps_list, |
|
mean_fps=mean_fps, |
|
ms_times_pre_image=times_pre_image_list, |
|
mean_times_pre_image=mean_times_pre_image) |
|
else: |
|
print( |
|
f'{cfg_path} fps : {fps:.{args.round_num}f} img / s, ' |
|
f'times per image: {1000 / fps:.{args.round_num}f} ' |
|
f'ms / img', |
|
flush=True) |
|
result_dict[cfg_path] = dict( |
|
fps=round(fps, args.round_num), |
|
ms_times_pre_image=round(1000 / fps, args.round_num)) |
|
except Exception as e: |
|
print(f'{cfg_path} error: {repr(e)}') |
|
if args.repeat_num > 1: |
|
result_dict[cfg_path] = dict( |
|
fps=[0], |
|
mean_fps=0, |
|
ms_times_pre_image=[0], |
|
mean_times_pre_image=0) |
|
else: |
|
result_dict[cfg_path] = dict(fps=0, ms_times_pre_image=0) |
|
|
|
if args.out: |
|
mkdir_or_exist(args.out) |
|
dump(result_dict, osp.join(args.out, 'batch_inference_fps.json')) |
|
|
|
results2markdown(result_dict) |
|
|
