|
- # Copyright (c) OpenMMLab. All rights reserved.
- import argparse
- import os.path as osp
-
- import mmcv
- import numpy as np
- from mmcv import Config, DictAction
-
- from mmdet.core.evaluation import eval_map
- from mmdet.core.visualization import imshow_gt_det_bboxes
- from mmdet.datasets import build_dataset, get_loading_pipeline
-
-
- def bbox_map_eval(det_result, annotation):
- """Evaluate mAP of single image det result.
-
- Args:
- det_result (list[list]): [[cls1_det, cls2_det, ...], ...].
- The outer list indicates images, and the inner list indicates
- per-class detected bboxes.
- annotation (dict): Ground truth annotations where keys of
- annotations are:
-
- - bboxes: numpy array of shape (n, 4)
- - labels: numpy array of shape (n, )
- - bboxes_ignore (optional): numpy array of shape (k, 4)
- - labels_ignore (optional): numpy array of shape (k, )
-
- Returns:
- float: mAP
- """
-
- # use only bbox det result
- if isinstance(det_result, tuple):
- bbox_det_result = [det_result[0]]
- else:
- bbox_det_result = [det_result]
- # mAP
- iou_thrs = np.linspace(
- .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True)
- mean_aps = []
- for thr in iou_thrs:
- mean_ap, _ = eval_map(
- bbox_det_result, [annotation], iou_thr=thr, logger='silent')
- mean_aps.append(mean_ap)
- return sum(mean_aps) / len(mean_aps)
-
-
- class ResultVisualizer:
- """Display and save evaluation results.
-
- Args:
- show (bool): Whether to show the image. Default: True
- wait_time (float): Value of waitKey param. Default: 0.
- score_thr (float): Minimum score of bboxes to be shown.
- Default: 0
- """
-
- def __init__(self, show=False, wait_time=0, score_thr=0):
- self.show = show
- self.wait_time = wait_time
- self.score_thr = score_thr
-
- def _save_image_gts_results(self, dataset, results, mAPs, out_dir=None):
- mmcv.mkdir_or_exist(out_dir)
-
- for mAP_info in mAPs:
- index, mAP = mAP_info
- data_info = dataset.prepare_train_img(index)
-
- # calc save file path
- filename = data_info['filename']
- if data_info['img_prefix'] is not None:
- filename = osp.join(data_info['img_prefix'], filename)
- else:
- filename = data_info['filename']
- fname, name = osp.splitext(osp.basename(filename))
- save_filename = fname + '_' + str(round(mAP, 3)) + name
- out_file = osp.join(out_dir, save_filename)
- imshow_gt_det_bboxes(
- data_info['img'],
- data_info,
- results[index],
- dataset.CLASSES,
- show=self.show,
- score_thr=self.score_thr,
- wait_time=self.wait_time,
- out_file=out_file)
-
- def evaluate_and_show(self,
- dataset,
- results,
- topk=20,
- show_dir='work_dir',
- eval_fn=None):
- """Evaluate and show results.
-
- Args:
- dataset (Dataset): A PyTorch dataset.
- results (list): Det results from test results pkl file
- topk (int): Number of the highest topk and
- lowest topk after evaluation index sorting. Default: 20
- show_dir (str, optional): The filename to write the image.
- Default: 'work_dir'
- eval_fn (callable, optional): Eval function, Default: None
- """
-
- assert topk > 0
- if (topk * 2) > len(dataset):
- topk = len(dataset) // 2
-
- if eval_fn is None:
- eval_fn = bbox_map_eval
- else:
- assert callable(eval_fn)
-
- prog_bar = mmcv.ProgressBar(len(results))
- _mAPs = {}
- for i, (result, ) in enumerate(zip(results)):
- # self.dataset[i] should not call directly
- # because there is a risk of mismatch
- data_info = dataset.prepare_train_img(i)
- mAP = eval_fn(result, data_info['ann_info'])
- _mAPs[i] = mAP
- prog_bar.update()
-
- # descending select topk image
- _mAPs = list(sorted(_mAPs.items(), key=lambda kv: kv[1]))
- good_mAPs = _mAPs[-topk:]
- bad_mAPs = _mAPs[:topk]
-
- good_dir = osp.abspath(osp.join(show_dir, 'good'))
- bad_dir = osp.abspath(osp.join(show_dir, 'bad'))
- self._save_image_gts_results(dataset, results, good_mAPs, good_dir)
- self._save_image_gts_results(dataset, results, bad_mAPs, bad_dir)
-
-
- def parse_args():
- parser = argparse.ArgumentParser(
- description='MMDet eval image prediction result for each')
- parser.add_argument('config', help='test config file path')
- parser.add_argument(
- 'prediction_path', help='prediction path where test pkl result')
- parser.add_argument(
- 'show_dir', help='directory where painted images will be saved')
- parser.add_argument('--show', action='store_true', help='show results')
- parser.add_argument(
- '--wait-time',
- type=float,
- default=0,
- help='the interval of show (s), 0 is block')
- parser.add_argument(
- '--topk',
- default=20,
- type=int,
- help='saved Number of the highest topk '
- 'and lowest topk after index sorting')
- parser.add_argument(
- '--show-score-thr',
- type=float,
- default=0,
- help='score threshold (default: 0.)')
- parser.add_argument(
- '--cfg-options',
- nargs='+',
- action=DictAction,
- help='override some settings in the used config, the key-value pair '
- 'in xxx=yyy format will be merged into config file. If the value to '
- 'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
- 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
- 'Note that the quotation marks are necessary and that no white space '
- 'is allowed.')
- args = parser.parse_args()
- return args
-
-
- def main():
- args = parse_args()
-
- mmcv.check_file_exist(args.prediction_path)
-
- cfg = Config.fromfile(args.config)
- if args.cfg_options is not None:
- cfg.merge_from_dict(args.cfg_options)
- cfg.data.test.test_mode = True
- # import modules from string list.
- if cfg.get('custom_imports', None):
- from mmcv.utils import import_modules_from_strings
- import_modules_from_strings(**cfg['custom_imports'])
-
- cfg.data.test.pop('samples_per_gpu', 0)
- cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.pipeline)
- dataset = build_dataset(cfg.data.test)
- outputs = mmcv.load(args.prediction_path)
-
- result_visualizer = ResultVisualizer(args.show, args.wait_time,
- args.show_score_thr)
- result_visualizer.evaluate_and_show(
- dataset, outputs, topk=args.topk, show_dir=args.show_dir)
-
-
- if __name__ == '__main__':
- main()
|