Skip to content

GitLab

Commit 879a6ae4 authored by Kai Chen's avatar Kai Chen
Browse files

bug fix and cleaning

parent 1e35964c
Hide whitespace changes
Inline Side-by-side
Showing with 248 additions and 417 deletions
tools/examples/r50_fpn_maskrcnn_1x.py → tools/configs/r50_fpn_maskrcnn_1x.py
View file @ 879a6ae4
# model settings # model settings
model = dict( model = dict(
pretrained= type='MaskRCNN',
'/mnt/lustre/pangjiangmiao/initmodel/pytorch/resnet50-19c8e357.pth', pretrained='modelzoo://resnet50',
backbone=dict( backbone=dict(
type='resnet', type='resnet',
depth=50, depth=50,
...@@ -25,7 +25,7 @@ model = dict( ...@@ -25,7 +25,7 @@ model = dict(
target_means=[.0, .0, .0, .0], target_means=[.0, .0, .0, .0],
target_stds=[1.0, 1.0, 1.0, 1.0], target_stds=[1.0, 1.0, 1.0, 1.0],
use_sigmoid_cls=True), use_sigmoid_cls=True),
roi_block=dict( bbox_roi_extractor=dict(
type='SingleLevelRoI', type='SingleLevelRoI',
roi_layer=dict(type='RoIAlign', out_size=7, sample_num=2), roi_layer=dict(type='RoIAlign', out_size=7, sample_num=2),
out_channels=256, out_channels=256,
...@@ -40,7 +40,7 @@ model = dict( ...@@ -40,7 +40,7 @@ model = dict(
target_means=[0., 0., 0., 0.], target_means=[0., 0., 0., 0.],
target_stds=[0.1, 0.1, 0.2, 0.2], target_stds=[0.1, 0.1, 0.2, 0.2],
reg_class_agnostic=False), reg_class_agnostic=False),
mask_block=dict( mask_roi_extractor=dict(
type='SingleLevelRoI', type='SingleLevelRoI',
roi_layer=dict(type='RoIAlign', out_size=14, sample_num=2), roi_layer=dict(type='RoIAlign', out_size=14, sample_num=2),
out_channels=256, out_channels=256,
...@@ -51,8 +51,9 @@ model = dict( ...@@ -51,8 +51,9 @@ model = dict(
in_channels=256, in_channels=256,
conv_out_channels=256, conv_out_channels=256,
num_classes=81)) num_classes=81))
meta_params = dict( # model training and testing settings
rpn_train_cfg=dict( train_cfg = dict(
rpn=dict(
pos_fraction=0.5, pos_fraction=0.5,
pos_balance_sampling=False, pos_balance_sampling=False,
neg_pos_ub=256, neg_pos_ub=256,
...@@ -65,14 +66,7 @@ meta_params = dict( ...@@ -65,14 +66,7 @@ meta_params = dict(
pos_weight=-1, pos_weight=-1,
smoothl1_beta=1 / 9.0, smoothl1_beta=1 / 9.0,
debug=False), debug=False),
rpn_test_cfg=dict( rcnn=dict(
nms_across_levels=False,
nms_pre=2000,
nms_post=2000,
max_num=2000,
nms_thr=0.7,
min_bbox_size=0),
rcnn_train_cfg=dict(
mask_size=28, mask_size=28,
pos_iou_thr=0.5, pos_iou_thr=0.5,
neg_iou_thr=0.5, neg_iou_thr=0.5,
...@@ -84,44 +78,52 @@ meta_params = dict( ...@@ -84,44 +78,52 @@ meta_params = dict(
neg_pos_ub=512, neg_pos_ub=512,
neg_balance_thr=0, neg_balance_thr=0,
pos_weight=-1, pos_weight=-1,
debug=False), debug=False))
rcnn_test_cfg=dict( test_cfg = dict(
rpn=dict(
nms_across_levels=False,
nms_pre=2000,
nms_post=2000,
max_num=2000,
nms_thr=0.7,
min_bbox_size=0),
rcnn=dict(
score_thr=1e-3, max_per_img=100, nms_thr=0.5, mask_thr_binary=0.5)) score_thr=1e-3, max_per_img=100, nms_thr=0.5, mask_thr_binary=0.5))
# dataset settings # dataset settings
data_root = '/mnt/lustre/pangjiangmiao/dataset/coco/' dataset_type = 'CocoDataset'
data_root = '../data/coco/'
img_norm_cfg = dict( img_norm_cfg = dict(
mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
img_per_gpu = 1 data = dict(
data_workers = 2 imgs_per_gpu=2,
train_dataset = dict( workers_per_gpu=2,
with_mask=True, train=dict(
ann_file=data_root + 'annotations/instances_train2017.json', type=dataset_type,
img_prefix=data_root + 'train2017/', ann_file=data_root + 'annotations/instances_train2017.json',
img_scale=(1333, 800), img_prefix=data_root + 'train2017/',
img_norm_cfg=img_norm_cfg, img_scale=(1333, 800),
size_divisor=32, img_norm_cfg=img_norm_cfg,
flip_ratio=0.5) size_divisor=32,
test_dataset = dict( flip_ratio=0.5),
ann_file=data_root + 'annotations/instances_val2017.json', test=dict(
img_prefix=data_root + 'val2017/', type=dataset_type,
img_scale=(1333, 800), ann_file=data_root + 'annotations/instances_val2017.json',
img_norm_cfg=img_norm_cfg, img_prefix=data_root + 'val2017/',
size_divisor=32) img_scale=(1333, 800),
flip_ratio=0,
img_norm_cfg=img_norm_cfg,
size_divisor=32))
# optimizer # optimizer
optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)
grad_clip_config = dict(grad_clip=True, max_norm=35, norm_type=2) optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
# learning policy # learning policy
lr_policy = dict( lr_config = dict(
policy='step', policy='step',
warmup='linear', warmup='linear',
warmup_iters=500, warmup_iters=500,
warmup_ratio=0.333, warmup_ratio=0.333,
step=[8, 11]) step=[8, 11])
max_epoch = 12
checkpoint_config = dict(interval=1) checkpoint_config = dict(interval=1)
dist_params = dict(backend='nccl', port='29500', master_ip='127.0.0.1')
# logging settings
log_level = 'INFO'
# yapf:disable # yapf:disable
log_config = dict( log_config = dict(
interval=50, interval=50,
...@@ -130,7 +132,12 @@ log_config = dict( ...@@ -130,7 +132,12 @@ log_config = dict(
# ('TensorboardLoggerHook', dict(log_dir=work_dir + '/log')), # ('TensorboardLoggerHook', dict(log_dir=work_dir + '/log')),
]) ])
# yapf:enable # yapf:enable
work_dir = './model/r50_fpn_mask_rcnn_1x' # runtime settings
total_epochs = 12
device_ids = range(8)
dist_params = dict(backend='nccl', port='29500', master_ip='127.0.0.1')
log_level = 'INFO'
work_dir = './work_dirs/fpn_mask_rcnn_r50_1x'
load_from = None load_from = None
resume_from = None resume_from = None
workflow = [('train', 1)] workflow = [('train', 1)]
tools/examples/r50_fpn_rpn_1x.py → tools/configs/r50_fpn_rpn_1x.py
View file @ 879a6ae4
# model settings # model settings
model = dict( model = dict(
pretrained= type='RPN',
'/mnt/lustre/pangjiangmiao/initmodel/pytorch/resnet50-19c8e357.pth', pretrained='modelzoo://resnet50',
backbone=dict( backbone=dict(
type='resnet', type='resnet',
depth=50, depth=50,
...@@ -25,8 +25,9 @@ model = dict( ...@@ -25,8 +25,9 @@ model = dict(
target_means=[.0, .0, .0, .0], target_means=[.0, .0, .0, .0],
target_stds=[1.0, 1.0, 1.0, 1.0], target_stds=[1.0, 1.0, 1.0, 1.0],
use_sigmoid_cls=True)) use_sigmoid_cls=True))
meta_params = dict( # model training and testing settings
rpn_train_cfg=dict( train_cfg = dict(
rpn=dict(
pos_fraction=0.5, pos_fraction=0.5,
pos_balance_sampling=False, pos_balance_sampling=False,
neg_pos_ub=256, neg_pos_ub=256,
...@@ -38,8 +39,9 @@ meta_params = dict( ...@@ -38,8 +39,9 @@ meta_params = dict(
min_pos_iou=1e-3, min_pos_iou=1e-3,
pos_weight=-1, pos_weight=-1,
smoothl1_beta=1 / 9.0, smoothl1_beta=1 / 9.0,
debug=False), debug=False))
rpn_test_cfg=dict( test_cfg = dict(
rpn=dict(
nms_across_levels=False, nms_across_levels=False,
nms_pre=2000, nms_pre=2000,
nms_post=2000, nms_post=2000,
...@@ -47,49 +49,61 @@ meta_params = dict( ...@@ -47,49 +49,61 @@ meta_params = dict(
nms_thr=0.7, nms_thr=0.7,
min_bbox_size=0)) min_bbox_size=0))
# dataset settings # dataset settings
data_root = '/mnt/lustre/pangjiangmiao/dataset/coco/' dataset_type = 'CocoDataset'
data_root = '../data/coco/'
img_norm_cfg = dict( img_norm_cfg = dict(
mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
img_per_gpu = 1 data = dict(
data_workers = 2 imgs_per_gpu=2,
train_dataset = dict( workers_per_gpu=2,
ann_file=data_root + 'annotations/instances_train2017.json', train=dict(
img_prefix=data_root + 'train2017/', type=dataset_type,
img_scale=(1333, 800), ann_file=data_root + 'annotations/instances_train2017.json',
img_norm_cfg=img_norm_cfg, img_prefix=data_root + 'train2017/',
size_divisor=32, img_scale=(1333, 800),
flip_ratio=0.5) img_norm_cfg=img_norm_cfg,
test_dataset = dict( size_divisor=32,
ann_file=data_root + 'annotations/instances_val2017.json', flip_ratio=0.5,
img_prefix=data_root + 'val2017/', with_mask=False,
img_scale=(1333, 800), with_crowd=False,
img_norm_cfg=img_norm_cfg, with_label=False,
size_divisor=32, test_mode=False),
test_mode=True) test=dict(
type=dataset_type,
ann_file=data_root + 'annotations/instances_val2017.json',
img_prefix=data_root + 'val2017/',
img_scale=(1333, 800),
flip_ratio=0,
img_norm_cfg=img_norm_cfg,
size_divisor=32,
with_mask=False,
with_label=False,
test_mode=True))
# optimizer # optimizer
optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)
grad_clip_config = dict(grad_clip=True, max_norm=35, norm_type=2) # runner configs
# learning policy optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
lr_policy = dict( lr_config = dict(
policy='step', policy='step',
warmup='linear', warmup='linear',
warmup_iters=500, warmup_iters=500,
warmup_ratio=0.333, warmup_ratio=1.0 / 3,
step=[8, 11]) step=[8, 11])
max_epoch = 12
checkpoint_config = dict(interval=1) checkpoint_config = dict(interval=1)
dist_params = dict(backend='nccl', port='29500', master_ip='127.0.0.1')
# logging settings
log_level = 'INFO'
# yapf:disable # yapf:disable
log_config = dict( log_config = dict(
interval=50, interval=50,
hooks=[ hooks=[
dict(type='TextLoggerHook'), dict(type='TextLoggerHook'),
# ('TensorboardLoggerHook', dict(log_dir=work_dir + '/log')), # dict(type='TensorboardLoggerHook', log_dir=work_dir + '/log')
]) ])
# yapf:enable # yapf:enable
work_dir = './model/r50_fpn_1x' # runtime settings
total_epochs = 12
device_ids = range(8)
dist_params = dict(backend='gloo', port='29500', master_ip='127.0.0.1')
log_level = 'INFO'
work_dir = './work_dirs/fpn_rpn_r50_1x'
load_from = None load_from = None
resume_from = None resume_from = None
workflow = [('train', 1)] workflow = [('train', 1)]
tools/dist_train.sh 0 → 100755
View file @ 879a6ae4
#!/usr/bin/env bash
PYTHON=${PYTHON:-"python"}
$PYTHON train.py $1 --dist --world-size $2 --rank 0 &
let MAX_RANK=$2-1
for i in `seq 1 $MAX_RANK`; do
$PYTHON train.py $1 --dist --world-size $2 --rank $i > /dev/null 2>&1 &
done
tools/eval.py deleted 100644 → 0
View file @ 1e35964c
from argparse import ArgumentParser
from multiprocessing import Pool
import matplotlib.pyplot as plt
import numpy as np
import copy
import os
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
def generate_area_range(splitRng=32, stop_size=128):
areaRng = [[0**2, 1e5**2], [0**2, 32**2], [32**2, 96**2], [96**2, 1e5**2]]
start = 0
while start < stop_size:
end = start + splitRng
areaRng.append([start * start, end * end])
start = end
areaRng.append([start * start, 1e5**2])
return areaRng
def print_summarize(iouThr=None,
iouThrs=None,
precision=None,
recall=None,
areaRng_id=4,
areaRngs=None,
maxDets_id=2,
maxDets=None):
assert (precision is not None) or (recall is not None)
iStr = ' {:<18} {} @[ IoU={:<9} | size={:>5}-{:>5} | maxDets={:>3d} ] = {:0.3f}'
titleStr = 'Average Precision' if precision is not None else 'Average Recall'
typeStr = '(AP)' if precision is not None else '(AR)'
iouStr = '{:0.2f}:{:0.2f}'.format(iouThrs[0], iouThrs[-1]) \
if iouThr is None else '{:0.2f}'.format(iouThr)
aind = [areaRng_id]
mind = [maxDets_id]
if precision is not None:
# dimension of precision: [TxRxKxAxM]
s = precision
# IoU
if iouThr is not None:
t = np.where(iouThr == iouThrs)[0]
s = s[t]
s = s[:, :, :, aind, mind]
else:
# dimension of recall: [TxKxAxM]
s = recall
if iouThr is not None:
t = np.where(iouThr == iouThrs)[0]
s = s[t]
s = s[:, :, aind, mind]
if len(s[s > -1]) == 0:
mean_s = -1
else:
mean_s = np.mean(s[s > -1])
print(
iStr.format(
titleStr, typeStr, iouStr, np.sqrt(areaRngs[areaRng_id][0]),
np.sqrt(areaRngs[areaRng_id][1])
if np.sqrt(areaRngs[areaRng_id][1]) < 999 else 'max',
maxDets[maxDets_id], mean_s))
def eval_results(res_file, ann_file, res_types, splitRng):
for res_type in res_types:
assert res_type in ['proposal', 'bbox', 'segm', 'keypoints']
areaRng = generate_area_range(splitRng)
cocoGt = COCO(ann_file)
cocoDt = cocoGt.loadRes(res_file)
imgIds = cocoGt.getImgIds()
for res_type in res_types:
iou_type = 'bbox' if res_type == 'proposal' else res_type
cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
cocoEval.params.imgIds = imgIds
if res_type == 'proposal':
cocoEval.params.useCats = 0
cocoEval.params.maxDets = [100, 300, 1000]
cocoEval.params.areaRng = areaRng
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
ps = cocoEval.eval['precision']
rc = cocoEval.eval['recall']
for i in range(len(areaRng)):
print_summarize(None, cocoEval.params.iouThrs, ps, None, i,
areaRng, 2, cocoEval.params.maxDets)
def makeplot(rs, ps, outDir, class_name):
cs = np.vstack([
np.ones((2, 3)),
np.array([.31, .51, .74]),
np.array([.75, .31, .30]),
np.array([.36, .90, .38]),
np.array([.50, .39, .64]),
np.array([1, .6, 0])
])
areaNames = ['all', 'small', 'medium', 'large']
types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN']
for i in range(len(areaNames)):
area_ps = ps[..., i, 0]
figure_tile = class_name + '-' + areaNames[i]
aps = [ps_.mean() for ps_ in area_ps]
ps_curve = [
ps_.mean(axis=1) if ps_.ndim > 1 else ps_ for ps_ in area_ps
]
ps_curve.insert(0, np.zeros(ps_curve[0].shape))
fig = plt.figure()
ax = plt.subplot(111)
for k in range(len(types)):
ax.plot(rs, ps_curve[k + 1], color=[0, 0, 0], linewidth=0.5)
ax.fill_between(
rs,
ps_curve[k],
ps_curve[k + 1],
color=cs[k],
label=str('[{:.3f}'.format(aps[k]) + ']' + types[k]))
plt.xlabel('recall')
plt.ylabel('precision')
plt.xlim(0, 1.)
plt.ylim(0, 1.)
plt.title(figure_tile)
plt.legend()
# plt.show()
fig.savefig(outDir + '/{}.png'.format(figure_tile))
plt.close(fig)
def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type):
nm = cocoGt.loadCats(catId)[0]
print('--------------analyzing {}-{}---------------'.format(
k + 1, nm['name']))
ps_ = {}
dt = copy.deepcopy(cocoDt)
nm = cocoGt.loadCats(catId)[0]
imgIds = cocoGt.getImgIds()
dt_anns = dt.dataset['annotations']
select_dt_anns = []
for ann in dt_anns:
if ann['category_id'] == catId:
select_dt_anns.append(ann)
dt.dataset['annotations'] = select_dt_anns
dt.createIndex()
# compute precision but ignore superclass confusion
gt = copy.deepcopy(cocoGt)
child_catIds = gt.getCatIds(supNms=[nm['supercategory']])
for idx, ann in enumerate(gt.dataset['annotations']):
if (ann['category_id'] in child_catIds
and ann['category_id'] != catId):
gt.dataset['annotations'][idx]['ignore'] = 1
gt.dataset['annotations'][idx]['iscrowd'] = 1
gt.dataset['annotations'][idx]['category_id'] = catId
cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
cocoEval.params.imgIds = imgIds
cocoEval.params.maxDets = [100]
cocoEval.params.iouThrs = [.1]
cocoEval.params.useCats = 1
cocoEval.evaluate()
cocoEval.accumulate()
ps_supercategory = cocoEval.eval['precision'][0, :, k, :, :]
ps_['ps_supercategory'] = ps_supercategory
# compute precision but ignore any class confusion
gt = copy.deepcopy(cocoGt)
for idx, ann in enumerate(gt.dataset['annotations']):
if ann['category_id'] != catId:
gt.dataset['annotations'][idx]['ignore'] = 1
gt.dataset['annotations'][idx]['iscrowd'] = 1
gt.dataset['annotations'][idx]['category_id'] = catId
cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type)
cocoEval.params.imgIds = imgIds
cocoEval.params.maxDets = [100]
cocoEval.params.iouThrs = [.1]
cocoEval.params.useCats = 1
cocoEval.evaluate()
cocoEval.accumulate()
ps_allcategory = cocoEval.eval['precision'][0, :, k, :, :]
ps_['ps_allcategory'] = ps_allcategory
return k, ps_
def analyze_results(res_file, ann_file, res_types, out_dir):
for res_type in res_types:
assert res_type in ['bbox', 'segm']
directory = os.path.dirname(out_dir + '/')
if not os.path.exists(directory):
print('-------------create {}-----------------'.format(out_dir))
os.makedirs(directory)
cocoGt = COCO(ann_file)
cocoDt = cocoGt.loadRes(res_file)
imgIds = cocoGt.getImgIds()
for res_type in res_types:
iou_type = res_type
cocoEval = COCOeval(
copy.deepcopy(cocoGt), copy.deepcopy(cocoDt), iou_type)
cocoEval.params.imgIds = imgIds
cocoEval.params.iouThrs = [.75, .5, .1]
cocoEval.params.maxDets = [100]
cocoEval.evaluate()
cocoEval.accumulate()
ps = cocoEval.eval['precision']
ps = np.vstack([ps, np.zeros((4, *ps.shape[1:]))])
catIds = cocoGt.getCatIds()
recThrs = cocoEval.params.recThrs
with Pool(processes=48) as pool:
args = [(k, cocoDt, cocoGt, catId, iou_type)
for k, catId in enumerate(catIds)]
analyze_results = pool.starmap(analyze_individual_category, args)
for k, catId in enumerate(catIds):
nm = cocoGt.loadCats(catId)[0]
print('--------------saving {}-{}---------------'.format(
k + 1, nm['name']))
analyze_result = analyze_results[k]
assert k == analyze_result[0]
ps_supercategory = analyze_result[1]['ps_supercategory']
ps_allcategory = analyze_result[1]['ps_allcategory']
# compute precision but ignore superclass confusion
ps[3, :, k, :, :] = ps_supercategory
# compute precision but ignore any class confusion
ps[4, :, k, :, :] = ps_allcategory
# fill in background and false negative errors and plot
ps[ps == -1] = 0
ps[5, :, k, :, :] = (ps[4, :, k, :, :] > 0)
ps[6, :, k, :, :] = 1.0
makeplot(recThrs, ps[:, :, k], out_dir, nm['name'])
makeplot(recThrs, ps, out_dir, 'all')
def main():
parser = ArgumentParser(description='COCO Evaluation')
parser.add_argument('result', help='result file path')
parser.add_argument(
'--ann',
default='/mnt/SSD/dataset/coco/annotations/instances_minival2017.json',
help='annotation file path')
parser.add_argument(
'--types', type=str, nargs='+', default=['bbox'], help='result types')
parser.add_argument(
'--analyze', action='store_true', help='whether to analyze results')
parser.add_argument(
'--out_dir',
type=str,
default=None,
help='dir to save analyze result images')
parser.add_argument(
'--splitRng',
type=int,
default=32,
help='range to split area in evaluation')
args = parser.parse_args()
if not args.analyze:
eval_results(args.result, args.ann, args.types, splitRng=args.splitRng)
else:
assert args.out_dir is not None
analyze_results(
args.result, args.ann, args.types, out_dir=args.out_dir)
if __name__ == '__main__':
main()
tools/test.py
View file @ 879a6ae4
import os.path as osp
import sys
sys.path.append(osp.abspath(osp.join(__file__, '../../')))
sys.path.append('/mnt/lustre/pangjiangmiao/sensenet_folder/mmcv')
import argparse import argparse
import numpy as np
import torch import torch
import mmcv import mmcv
from mmcv import Config from mmcv.torchpack import load_checkpoint, parallel_test, obj_from_dict
from mmcv.torchpack import load_checkpoint, parallel_test
from mmdet.core import _data_func, results2json from mmdet import datasets
from mmdet.datasets import CocoDataset from mmdet.core import results2json, coco_eval
from mmdet.datasets.data_engine import build_data from mmdet.datasets.loader import collate, build_dataloader
from mmdet.models import Detector from mmdet.models import build_detector, detectors
from mmdet.nn.parallel import scatter, MMDataParallel
def single_test(model, data_loader, show=False):
model.eval()
results = []
prog_bar = mmcv.ProgressBar(len(data_loader.dataset))
for i, data in enumerate(data_loader):
with torch.no_grad():
result = model(**data, return_loss=False, rescale=not show)
results.append(result)
if show:
model.module.show_result(data, result,
data_loader.dataset.img_norm_cfg)
batch_size = data['img'][0].size(0)
for _ in range(batch_size):
prog_bar.update()
return results
def _data_func(data, device_id):
data = scatter(collate([data], samples_per_gpu=1), [device_id])[0]
return dict(**data, return_loss=False, rescale=True)
def parse_args(): def parse_args():
parser = argparse.ArgumentParser(description='MMDet test detector') parser = argparse.ArgumentParser(description='MMDet test detector')
parser.add_argument('config', help='test config file path') parser.add_argument('config', help='test config file path')
parser.add_argument('checkpoint', help='checkpoint file') parser.add_argument('checkpoint', help='checkpoint file')
parser.add_argument('--world_size', default=1, type=int) parser.add_argument('--gpus', default=1, type=int)
parser.add_argument('--out', help='output result file') parser.add_argument('--out', help='output result file')
parser.add_argument( parser.add_argument(
'--out_json', action='store_true', help='get json output file') '--eval',
type=str,
nargs='+',
choices=['proposal', 'bbox', 'segm', 'keypoints'],
help='eval types')
parser.add_argument('--show', action='store_true', help='show results')
args = parser.parse_args() args = parser.parse_args()
return args return args
...@@ -32,33 +56,38 @@ args = parse_args() ...@@ -32,33 +56,38 @@ args = parse_args()
def main(): def main():
cfg = Config.fromfile(args.config) cfg = mmcv.Config.fromfile(args.config)
cfg.model['pretrained'] = None cfg.model.pretrained = None
# TODO this img_per_gpu cfg.data.test.test_mode = True
cfg.img_per_gpu == 1
dataset = obj_from_dict(cfg.data.test, datasets)
if args.world_size == 1: if args.gpus == 1:
# TODO verify this part model = build_detector(
args.dist = False cfg.model, train_cfg=None, test_cfg=cfg.test_cfg)
args.img_per_gpu = cfg.img_per_gpu
args.data_workers = cfg.data_workers
model = Detector(**cfg.model, **meta_params)
load_checkpoint(model, args.checkpoint) load_checkpoint(model, args.checkpoint)
test_loader = build_data(cfg.test_dataset, args) model = MMDataParallel(model, device_ids=[0])
model = torch.nn.DataParallel(model, device_ids=0)
# TODO write single_test data_loader = build_dataloader(
outputs = single_test(test_loader, model) dataset,
imgs_per_gpu=1,
workers_per_gpu=cfg.data.workers_per_gpu,
num_gpus=1,
dist=False,
shuffle=False)
outputs = single_test(model, data_loader, args.show)
else: else:
test_dataset = CocoDataset(**cfg.test_dataset) model_args = cfg.model.copy()
model = dict(cfg.model, **cfg.meta_params) model_args.update(train_cfg=None, test_cfg=cfg.test_cfg)
outputs = parallel_test(Detector, model, model_type = getattr(detectors, model_args.pop('type'))
args.checkpoint, test_dataset, _data_func, outputs = parallel_test(model_type, model_args, args.checkpoint,
range(args.world_size)) dataset, _data_func, range(args.gpus))
if args.out: if args.out:
mmcv.dump(outputs, args.out, protocol=4) mmcv.dump(outputs, args.out)
if args.out_json: if args.eval:
results2json(test_dataset, outputs, args.out + '.json') json_file = args.out + '.json'
results2json(dataset, outputs, json_file)
coco_eval(json_file, args.eval, dataset.coco)
if __name__ == '__main__': if __name__ == '__main__':
......
tools/train.py
View file @ 879a6ae4
from __future__ import division from __future__ import division
import argparse import argparse
import sys from collections import OrderedDict
import os.path as osp
sys.path.append(osp.abspath(osp.join(__file__, '../../')))
sys.path.append('/mnt/lustre/pangjiangmiao/sensenet_folder/mmcv')
import torch import torch
import torch.multiprocessing as mp
from mmcv import Config from mmcv import Config
from mmcv.torchpack import Runner from mmcv.torchpack import Runner, obj_from_dict
from mmdet.core import (batch_processor, init_dist, broadcast_params,
DistOptimizerStepperHook, DistSamplerSeedHook) from mmdet import datasets
from mmdet.datasets.data_engine import build_data from mmdet.core import init_dist, DistOptimizerHook, DistSamplerSeedHook
from mmdet.models import Detector from mmdet.datasets.loader import build_dataloader
from mmdet.nn.parallel import MMDataParallel from mmdet.models import build_detector
from mmdet.nn.parallel import MMDataParallel, MMDistributedDataParallel
def parse_losses(losses):
log_vars = OrderedDict()
for loss_name, loss_value in losses.items():
if isinstance(loss_value, torch.Tensor):
log_vars[loss_name] = loss_value.mean()
elif isinstance(loss_value, list):
log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value)
else:
raise TypeError(
'{} is not a tensor or list of tensors'.format(loss_name))
loss = sum(_value for _key, _value in log_vars.items() if 'loss' in _key)
log_vars['loss'] = loss
for name in log_vars:
log_vars[name] = log_vars[name].item()
return loss, log_vars
def batch_processor(model, data, train_mode, args=None):
losses = model(**data)
loss, log_vars = parse_losses(losses)
outputs = dict(
loss=loss / args.world_size,
log_vars=log_vars,
num_samples=len(data['img'].data))
return outputs
def parse_args(): def parse_args():
parser = argparse.ArgumentParser(description='MMDet train val detector') parser = argparse.ArgumentParser(description='Train a detector')
parser.add_argument('config', help='train config file path') parser.add_argument('config', help='train config file path')
parser.add_argument('--validate', action='store_true', help='validate')
parser.add_argument( parser.add_argument(
'--dist', action='store_true', help='distributed training or not') '--validate',
parser.add_argument('--world_size', default=1, type=int) action='store_true',
help='whether to add a validate phase')
parser.add_argument(
'--dist', action='store_true', help='use distributed training or not')
parser.add_argument('--world-size', default=1, type=int)
parser.add_argument('--rank', default=0, type=int) parser.add_argument('--rank', default=0, type=int)
args = parser.parse_args() args = parser.parse_args()
...@@ -33,53 +66,57 @@ args = parse_args() ...@@ -33,53 +66,57 @@ args = parse_args()
def main(): def main():
# Enable distributed training or not # get config from file
cfg = Config.fromfile(args.config)
cfg.update(world_size=args.world_size, rank=args.rank)
# init distributed environment if necessary
if args.dist: if args.dist:
print('Enable distributed training.') print('Enable distributed training.')
mp.set_start_method("spawn", force=True) init_dist(args.world_size, args.rank, **cfg.dist_params)
init_dist(
args.world_size,
args.rank,
**cfg.dist_params)
else: else:
print('Disabled distributed training.') print('Disabled distributed training.')
# Fetch config information # prepare data loaders
cfg = Config.fromfile(args.config) train_dataset = obj_from_dict(cfg.data.train, datasets)
# TODO more flexible data_loaders = [
args.img_per_gpu = cfg.img_per_gpu build_dataloader(
args.data_workers = cfg.data_workers train_dataset, cfg.data.imgs_per_gpu, cfg.data.workers_per_gpu,
len(cfg.device_ids), args.dist, cfg.world_size, cfg.rank)
# prepare training loader ]
train_loader = [build_data(cfg.train_dataset, args)]
if args.validate: if args.validate:
val_loader = build_data(cfg.val_dataset, args) val_dataset = obj_from_dict(cfg.data.val, datasets)
train_loader.append(val_loader) data_loaders.append(
build_dataloader(
val_dataset, cfg.data.imgs_per_gpu, cfg.data.workers_per_gpu,
len(cfg.device_ids), args.dist, cfg.world_size, cfg.rank))
# build model # build model
model = Detector(**cfg.model, **cfg.meta_params) model = build_detector(
cfg.model, train_cfg=cfg.train_cfg, test_cfg=cfg.test_cfg)
if args.dist: if args.dist:
model = model.cuda() model = MMDistributedDataParallel(
broadcast_params(model) model, device_ids=[cfg.rank], broadcast_buffers=False).cuda()
else: else:
device_ids = args.rank % torch.cuda.device_count() model = MMDataParallel(model, device_ids=cfg.device_ids).cuda()
model = MMDataParallel(model, device_ids=device_ids).cuda()
# register hooks # build runner
runner = Runner(model, batch_processor, cfg.optimizer, cfg.work_dir, runner = Runner(model, batch_processor, cfg.optimizer, cfg.work_dir,
cfg.log_level) cfg.log_level)
optimizer_stepper = DistOptimizerStepperHook( # register hooks
**cfg.grad_clip_config) if args.dist else cfg.grad_clip_config optimizer_config = DistOptimizerHook(
runner.register_training_hooks(cfg.lr_policy, optimizer_stepper, **cfg.optimizer_config) if args.dist else cfg.optimizer_config
runner.register_training_hooks(cfg.lr_config, optimizer_config,
cfg.checkpoint_config, cfg.log_config) cfg.checkpoint_config, cfg.log_config)
if args.dist: if args.dist:
runner.register_hook(DistSamplerSeedHook()) runner.register_hook(DistSamplerSeedHook())
if cfg.resume_from: if cfg.resume_from:
runner.resume(cfg.resume_from) runner.resume(cfg.resume_from)
elif cfg.load_from: elif cfg.load_from:
runner.load_checkpoint(cfg.load_from) runner.load_checkpoint(cfg.load_from)
runner.run(train_loader, cfg.workflow, cfg.max_epoch, args=args) runner.run(data_loaders, cfg.workflow, cfg.total_epochs, args=args)
if __name__ == "__main__": if __name__ == '__main__':
main() main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment