Skip to content

GitLab

Unverified Commit 44fefe60 authored by Anirudh's avatar Anirudh Committed by GitHub
Browse files

Port test_models to pytest (#3978)

parent a2329ff6
Hide whitespace changes
Inline Side-by-side
Showing with 371 additions and 369 deletions
test/test_models.py
View file @ 44fefe60
import os import os
import io import io
import sys import sys
from common_utils import TestCase, map_nested_tensor_object, freeze_rng_state, set_rng_seed from common_utils import map_nested_tensor_object, freeze_rng_state, set_rng_seed, cpu_and_gpu, needs_cuda, cpu_only
from _utils_internal import get_relative_path from _utils_internal import get_relative_path
from collections import OrderedDict from collections import OrderedDict
from itertools import product
import functools import functools
import operator import operator
import torch import torch
import torch.nn as nn import torch.nn as nn
from torchvision import models from torchvision import models
import unittest
import warnings
import pytest import pytest
import warnings
ACCEPT = os.getenv('EXPECTTEST_ACCEPT', '0') == '1' ACCEPT = os.getenv('EXPECTTEST_ACCEPT', '0') == '1'
...@@ -220,399 +217,404 @@ _model_params = { ...@@ -220,399 +217,404 @@ _model_params = {
} }
class ModelTester(TestCase): def _make_sliced_model(model, stop_layer):
def _test_classification_model(self, name, dev): layers = OrderedDict()
set_rng_seed(0) for name, layer in model.named_children():
defaults = { layers[name] = layer
'num_classes': 50, if name == stop_layer:
'input_shape': (1, 3, 224, 224), break
} new_model = torch.nn.Sequential(layers)
kwargs = {**defaults, **_model_params.get(name, {})} return new_model
input_shape = kwargs.pop('input_shape')
model = models.__dict__[name](**kwargs) @cpu_only
model.eval().to(device=dev) @pytest.mark.parametrize('model_name', ['densenet121', 'densenet169', 'densenet201', 'densenet161'])
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests def test_memory_efficient_densenet(model_name):
x = torch.rand(input_shape).to(device=dev) input_shape = (1, 3, 300, 300)
out = model(x) x = torch.rand(input_shape)
_assert_expected(out.cpu(), name, prec=0.1)
self.assertEqual(out.shape[-1], 50) model1 = models.__dict__[model_name](num_classes=50, memory_efficient=True)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None)) params = model1.state_dict()
num_params = sum([x.numel() for x in model1.parameters()])
if dev == torch.device("cuda"): model1.eval()
with torch.cuda.amp.autocast(): out1 = model1(x)
out = model(x) out1.sum().backward()
# See autocast_flaky_numerics comment at top of file. num_grad = sum([x.grad.numel() for x in model1.parameters() if x.grad is not None])
if name not in autocast_flaky_numerics:
_assert_expected(out.cpu(), name, prec=0.1) model2 = models.__dict__[model_name](num_classes=50, memory_efficient=False)
self.assertEqual(out.shape[-1], 50) model2.load_state_dict(params)
model2.eval()
def _test_segmentation_model(self, name, dev): out2 = model2(x)
set_rng_seed(0)
defaults = { assert num_params == num_grad
'num_classes': 10, torch.testing.assert_close(out1, out2, rtol=0.0, atol=1e-5)
'pretrained_backbone': False,
'input_shape': (1, 3, 32, 32),
} @cpu_only
kwargs = {**defaults, **_model_params.get(name, {})} @pytest.mark.parametrize('dilate_layer_2', (True, False))
input_shape = kwargs.pop('input_shape') @pytest.mark.parametrize('dilate_layer_3', (True, False))
@pytest.mark.parametrize('dilate_layer_4', (True, False))
model = models.segmentation.__dict__[name](**kwargs) def test_resnet_dilation(dilate_layer_2, dilate_layer_3, dilate_layer_4):
model.eval().to(device=dev) # TODO improve tests to also check that each layer has the right dimensionality
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests model = models.__dict__["resnet50"](replace_stride_with_dilation=(dilate_layer_2, dilate_layer_3, dilate_layer_4))
x = torch.rand(input_shape).to(device=dev) model = _make_sliced_model(model, stop_layer="layer4")
out = model(x)["out"] model.eval()
x = torch.rand(1, 3, 224, 224)
def check_out(out): out = model(x)
prec = 0.01 f = 2 ** sum((dilate_layer_2, dilate_layer_3, dilate_layer_4))
try: assert out.shape == (1, 2048, 7 * f, 7 * f)
# We first try to assert the entire output if possible. This is not
# only the best way to assert results but also handles the cases
# where we need to create a new expected result. @cpu_only
_assert_expected(out.cpu(), name, prec=prec) def test_mobilenet_v2_residual_setting():
except AssertionError: model = models.__dict__["mobilenet_v2"](inverted_residual_setting=[[1, 16, 1, 1], [6, 24, 2, 2]])
# Unfortunately some segmentation models are flaky with autocast model.eval()
# so instead of validating the probability scores, check that the class x = torch.rand(1, 3, 224, 224)
# predictions match. out = model(x)
expected_file = _get_expected_file(name) assert out.shape[-1] == 1000
expected = torch.load(expected_file)
torch.testing.assert_close(out.argmax(dim=1), expected.argmax(dim=1), rtol=prec, atol=prec)
return False # Partial validation performed @cpu_only
@pytest.mark.parametrize('model_name', ["mobilenet_v2", "mobilenet_v3_large", "mobilenet_v3_small"])
return True # Full validation performed def test_mobilenet_norm_layer(model_name):
model = models.__dict__[model_name]()
full_validation = check_out(out) assert any(isinstance(x, nn.BatchNorm2d) for x in model.modules())
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None)) def get_gn(num_channels):
return nn.GroupNorm(32, num_channels)
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast(): model = models.__dict__[model_name](norm_layer=get_gn)
out = model(x)["out"] assert not(any(isinstance(x, nn.BatchNorm2d) for x in model.modules()))
# See autocast_flaky_numerics comment at top of file. assert any(isinstance(x, nn.GroupNorm) for x in model.modules())
if name not in autocast_flaky_numerics:
full_validation &= check_out(out)
@cpu_only
if not full_validation: def test_inception_v3_eval():
msg = "The output of {} could only be partially validated. " \ # replacement for models.inception_v3(pretrained=True) that does not download weights
"This is likely due to unit-test flakiness, but you may " \ kwargs = {}
"want to do additional manual checks if you made " \ kwargs['transform_input'] = True
"significant changes to the codebase.".format(self._testMethodName) kwargs['aux_logits'] = True
warnings.warn(msg, RuntimeWarning) kwargs['init_weights'] = False
raise unittest.SkipTest(msg) name = "inception_v3"
model = models.Inception3(**kwargs)
def _test_detection_model(self, name, dev): model.aux_logits = False
set_rng_seed(0) model.AuxLogits = None
defaults = { model = model.eval()
'num_classes': 50, x = torch.rand(1, 3, 299, 299)
'pretrained_backbone': False, _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
'input_shape': (3, 300, 300),
}
kwargs = {**defaults, **_model_params.get(name, {})} @cpu_only
input_shape = kwargs.pop('input_shape') def test_fasterrcnn_double():
model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
model = models.detection.__dict__[name](**kwargs) model.double()
model.eval().to(device=dev) model.eval()
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests input_shape = (3, 300, 300)
x = torch.rand(input_shape).to(device=dev) x = torch.rand(input_shape, dtype=torch.float64)
model_input = [x] model_input = [x]
out = model(model_input) out = model(model_input)
self.assertIs(model_input[0], x) assert model_input[0] is x
assert len(out) == 1
def check_out(out): assert "boxes" in out[0]
self.assertEqual(len(out), 1) assert "scores" in out[0]
assert "labels" in out[0]
def compact(tensor):
size = tensor.size()
elements_per_sample = functools.reduce(operator.mul, size[1:], 1) @cpu_only
if elements_per_sample > 30: def test_googlenet_eval():
return compute_mean_std(tensor) # replacement for models.googlenet(pretrained=True) that does not download weights
else: kwargs = {}
return subsample_tensor(tensor) kwargs['transform_input'] = True
kwargs['aux_logits'] = True
def subsample_tensor(tensor): kwargs['init_weights'] = False
num_elems = tensor.size(0) name = "googlenet"
num_samples = 20 model = models.GoogLeNet(**kwargs)
if num_elems <= num_samples: model.aux_logits = False
return tensor model.aux1 = None
model.aux2 = None
ith_index = num_elems // num_samples model = model.eval()
return tensor[ith_index - 1::ith_index] x = torch.rand(1, 3, 224, 224)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
def compute_mean_std(tensor):
# can't compute mean of integral tensor
tensor = tensor.to(torch.double) @needs_cuda
mean = torch.mean(tensor) def test_fasterrcnn_switch_devices():
std = torch.std(tensor) def checkOut(out):
return {"mean": mean, "std": std} assert len(out) == 1
assert "boxes" in out[0]
output = map_nested_tensor_object(out, tensor_map_fn=compact) assert "scores" in out[0]
prec = 0.01 assert "labels" in out[0]
try:
# We first try to assert the entire output if possible. This is not model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
# only the best way to assert results but also handles the cases model.cuda()
# where we need to create a new expected result. model.eval()
_assert_expected(output, name, prec=prec) input_shape = (3, 300, 300)
except AssertionError: x = torch.rand(input_shape, device='cuda')
# Unfortunately detection models are flaky due to the unstable sort model_input = [x]
# in NMS. If matching across all outputs fails, use the same approach out = model(model_input)
# as in NMSTester.test_nms_cuda to see if this is caused by duplicate assert model_input[0] is x
# scores.
expected_file = _get_expected_file(name) checkOut(out)
expected = torch.load(expected_file)
torch.testing.assert_close(output[0]["scores"], expected[0]["scores"], rtol=prec, atol=prec, with torch.cuda.amp.autocast():
check_device=False, check_dtype=False)
# Note: Fmassa proposed turning off NMS by adapting the threshold
# and then using the Hungarian algorithm as in DETR to find the
# best match between output and expected boxes and eliminate some
# of the flakiness. Worth exploring.
return False # Partial validation performed
return True # Full validation performed
full_validation = check_out(out)
_check_jit_scriptable(model, ([x],), unwrapper=script_model_unwrapper.get(name, None))
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast():
out = model(model_input)
# See autocast_flaky_numerics comment at top of file.
if name not in autocast_flaky_numerics:
full_validation &= check_out(out)
if not full_validation:
msg = "The output of {} could only be partially validated. " \
"This is likely due to unit-test flakiness, but you may " \
"want to do additional manual checks if you made " \
"significant changes to the codebase.".format(self._testMethodName)
warnings.warn(msg, RuntimeWarning)
raise unittest.SkipTest(msg)
def _test_detection_model_validation(self, name):
set_rng_seed(0)
model = models.detection.__dict__[name](num_classes=50, pretrained_backbone=False)
input_shape = (3, 300, 300)
x = [torch.rand(input_shape)]
# validate that targets are present in training
self.assertRaises(ValueError, model, x)
# validate type
targets = [{'boxes': 0.}]
self.assertRaises(ValueError, model, x, targets=targets)
# validate boxes shape
for boxes in (torch.rand((4,)), torch.rand((1, 5))):
targets = [{'boxes': boxes}]
self.assertRaises(ValueError, model, x, targets=targets)
# validate that no degenerate boxes are present
boxes = torch.tensor([[1, 3, 1, 4], [2, 4, 3, 4]])
targets = [{'boxes': boxes}]
self.assertRaises(ValueError, model, x, targets=targets)
def _test_video_model(self, name, dev):
# the default input shape is
# bs * num_channels * clip_len * h *w
input_shape = (1, 3, 4, 112, 112)
# test both basicblock and Bottleneck
model = models.video.__dict__[name](num_classes=50)
model.eval().to(device=dev)
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
x = torch.rand(input_shape).to(device=dev)
out = model(x)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
self.assertEqual(out.shape[-1], 50)
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast():
out = model(x)
self.assertEqual(out.shape[-1], 50)
def _make_sliced_model(self, model, stop_layer):
layers = OrderedDict()
for name, layer in model.named_children():
layers[name] = layer
if name == stop_layer:
break
new_model = torch.nn.Sequential(layers)
return new_model
def test_memory_efficient_densenet(self):
input_shape = (1, 3, 300, 300)
x = torch.rand(input_shape)
for name in ['densenet121', 'densenet169', 'densenet201', 'densenet161']:
model1 = models.__dict__[name](num_classes=50, memory_efficient=True)
params = model1.state_dict()
num_params = sum([x.numel() for x in model1.parameters()])
model1.eval()
out1 = model1(x)
out1.sum().backward()
num_grad = sum([x.grad.numel() for x in model1.parameters() if x.grad is not None])
model2 = models.__dict__[name](num_classes=50, memory_efficient=False)
model2.load_state_dict(params)
model2.eval()
out2 = model2(x)
self.assertTrue(num_params == num_grad)
torch.testing.assert_close(out1, out2, rtol=0.0, atol=1e-5)
def test_resnet_dilation(self):
# TODO improve tests to also check that each layer has the right dimensionality
for i in product([False, True], [False, True], [False, True]):
model = models.__dict__["resnet50"](replace_stride_with_dilation=i)
model = self._make_sliced_model(model, stop_layer="layer4")
model.eval()
x = torch.rand(1, 3, 224, 224)
out = model(x)
f = 2 ** sum(i)
self.assertEqual(out.shape, (1, 2048, 7 * f, 7 * f))
def test_mobilenet_v2_residual_setting(self):
model = models.__dict__["mobilenet_v2"](inverted_residual_setting=[[1, 16, 1, 1], [6, 24, 2, 2]])
model.eval()
x = torch.rand(1, 3, 224, 224)
out = model(x)
self.assertEqual(out.shape[-1], 1000)
def test_mobilenet_norm_layer(self):
for name in ["mobilenet_v2", "mobilenet_v3_large", "mobilenet_v3_small"]:
model = models.__dict__[name]()
self.assertTrue(any(isinstance(x, nn.BatchNorm2d) for x in model.modules()))
def get_gn(num_channels):
return nn.GroupNorm(32, num_channels)
model = models.__dict__[name](norm_layer=get_gn)
self.assertFalse(any(isinstance(x, nn.BatchNorm2d) for x in model.modules()))
self.assertTrue(any(isinstance(x, nn.GroupNorm) for x in model.modules()))
def test_inception_v3_eval(self):
# replacement for models.inception_v3(pretrained=True) that does not download weights
kwargs = {}
kwargs['transform_input'] = True
kwargs['aux_logits'] = True
kwargs['init_weights'] = False
name = "inception_v3"
model = models.Inception3(**kwargs)
model.aux_logits = False
model.AuxLogits = None
model = model.eval()
x = torch.rand(1, 3, 299, 299)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
def test_fasterrcnn_double(self):
model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
model.double()
model.eval()
input_shape = (3, 300, 300)
x = torch.rand(input_shape, dtype=torch.float64)
model_input = [x]
out = model(model_input) out = model(model_input)
self.assertIs(model_input[0], x)
self.assertEqual(len(out), 1)
self.assertTrue("boxes" in out[0])
self.assertTrue("scores" in out[0])
self.assertTrue("labels" in out[0])
def test_googlenet_eval(self):
# replacement for models.googlenet(pretrained=True) that does not download weights
kwargs = {}
kwargs['transform_input'] = True
kwargs['aux_logits'] = True
kwargs['init_weights'] = False
name = "googlenet"
model = models.GoogLeNet(**kwargs)
model.aux_logits = False
model.aux1 = None
model.aux2 = None
model = model.eval()
x = torch.rand(1, 3, 224, 224)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
@unittest.skipIf(not torch.cuda.is_available(), 'needs GPU')
def test_fasterrcnn_switch_devices(self):
def checkOut(out):
self.assertEqual(len(out), 1)
self.assertTrue("boxes" in out[0])
self.assertTrue("scores" in out[0])
self.assertTrue("labels" in out[0])
model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
model.cuda()
model.eval()
input_shape = (3, 300, 300)
x = torch.rand(input_shape, device='cuda')
model_input = [x]
out = model(model_input)
self.assertIs(model_input[0], x)
checkOut(out)
with torch.cuda.amp.autocast(): checkOut(out)
out = model(model_input)
checkOut(out)
# now switch to cpu and make sure it works
model.cpu()
x = x.cpu()
out_cpu = model([x])
checkOut(out_cpu) # now switch to cpu and make sure it works
model.cpu()
x = x.cpu()
out_cpu = model([x])
def test_generalizedrcnn_transform_repr(self): checkOut(out_cpu)
min_size, max_size = 224, 299
image_mean = [0.485, 0.456, 0.406]
image_std = [0.229, 0.224, 0.225]
t = models.detection.transform.GeneralizedRCNNTransform(min_size=min_size, @cpu_only
max_size=max_size, def test_generalizedrcnn_transform_repr():
image_mean=image_mean,
image_std=image_std)
# Check integrity of object __repr__ attribute min_size, max_size = 224, 299
expected_string = 'GeneralizedRCNNTransform(' image_mean = [0.485, 0.456, 0.406]
_indent = '\n ' image_std = [0.229, 0.224, 0.225]
expected_string += '{0}Normalize(mean={1}, std={2})'.format(_indent, image_mean, image_std)
expected_string += '{0}Resize(min_size=({1},), max_size={2}, '.format(_indent, min_size, max_size)
expected_string += "mode='bilinear')\n)"
self.assertEqual(t.__repr__(), expected_string)
t = models.detection.transform.GeneralizedRCNNTransform(min_size=min_size,
max_size=max_size,
image_mean=image_mean,
image_std=image_std)
_devs = [torch.device("cpu"), torch.device("cuda")] if torch.cuda.is_available() else [torch.device("cpu")] # Check integrity of object __repr__ attribute
expected_string = 'GeneralizedRCNNTransform('
_indent = '\n '
expected_string += '{0}Normalize(mean={1}, std={2})'.format(_indent, image_mean, image_std)
expected_string += '{0}Resize(min_size=({1},), max_size={2}, '.format(_indent, min_size, max_size)
expected_string += "mode='bilinear')\n)"
assert t.__repr__() == expected_string
@pytest.mark.parametrize('model_name', get_available_classification_models()) @pytest.mark.parametrize('model_name', get_available_classification_models())
@pytest.mark.parametrize('dev', _devs) @pytest.mark.parametrize('dev', cpu_and_gpu())
def test_classification_model(model_name, dev): def test_classification_model(model_name, dev):
ModelTester()._test_classification_model(model_name, dev) set_rng_seed(0)
defaults = {
'num_classes': 50,
'input_shape': (1, 3, 224, 224),
}
kwargs = {**defaults, **_model_params.get(model_name, {})}
input_shape = kwargs.pop('input_shape')
model = models.__dict__[model_name](**kwargs)
model.eval().to(device=dev)
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
x = torch.rand(input_shape).to(device=dev)
out = model(x)
_assert_expected(out.cpu(), model_name, prec=0.1)
assert out.shape[-1] == 50
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None))
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast():
out = model(x)
# See autocast_flaky_numerics comment at top of file.
if model_name not in autocast_flaky_numerics:
_assert_expected(out.cpu(), model_name, prec=0.1)
assert out.shape[-1] == 50
@pytest.mark.parametrize('model_name', get_available_segmentation_models()) @pytest.mark.parametrize('model_name', get_available_segmentation_models())
@pytest.mark.parametrize('dev', _devs) @pytest.mark.parametrize('dev', cpu_and_gpu())
def test_segmentation_model(model_name, dev): def test_segmentation_model(model_name, dev):
ModelTester()._test_segmentation_model(model_name, dev) set_rng_seed(0)
defaults = {
'num_classes': 10,
'pretrained_backbone': False,
'input_shape': (1, 3, 32, 32),
}
kwargs = {**defaults, **_model_params.get(model_name, {})}
input_shape = kwargs.pop('input_shape')
model = models.segmentation.__dict__[model_name](**kwargs)
model.eval().to(device=dev)
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
x = torch.rand(input_shape).to(device=dev)
out = model(x)["out"]
def check_out(out):
prec = 0.01
try:
# We first try to assert the entire output if possible. This is not
# only the best way to assert results but also handles the cases
# where we need to create a new expected result.
_assert_expected(out.cpu(), model_name, prec=prec)
except AssertionError:
# Unfortunately some segmentation models are flaky with autocast
# so instead of validating the probability scores, check that the class
# predictions match.
expected_file = _get_expected_file(model_name)
expected = torch.load(expected_file)
torch.testing.assert_close(out.argmax(dim=1), expected.argmax(dim=1), rtol=prec, atol=prec)
return False # Partial validation performed
return True # Full validation performed
full_validation = check_out(out)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None))
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast():
out = model(x)["out"]
# See autocast_flaky_numerics comment at top of file.
if model_name not in autocast_flaky_numerics:
full_validation &= check_out(out)
if not full_validation:
msg = "The output of {} could only be partially validated. " \
"This is likely due to unit-test flakiness, but you may " \
"want to do additional manual checks if you made " \
"significant changes to the codebase.".format(test_segmentation_model.__name__)
warnings.warn(msg, RuntimeWarning)
pytest.skip(msg)
@pytest.mark.parametrize('model_name', get_available_detection_models()) @pytest.mark.parametrize('model_name', get_available_detection_models())
@pytest.mark.parametrize('dev', _devs) @pytest.mark.parametrize('dev', cpu_and_gpu())
def test_detection_model(model_name, dev): def test_detection_model(model_name, dev):
ModelTester()._test_detection_model(model_name, dev) set_rng_seed(0)
defaults = {
'num_classes': 50,
'pretrained_backbone': False,
'input_shape': (3, 300, 300),
}
kwargs = {**defaults, **_model_params.get(model_name, {})}
input_shape = kwargs.pop('input_shape')
model = models.detection.__dict__[model_name](**kwargs)
model.eval().to(device=dev)
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
x = torch.rand(input_shape).to(device=dev)
model_input = [x]
out = model(model_input)
assert model_input[0] is x
def check_out(out):
assert len(out) == 1
def compact(tensor):
size = tensor.size()
elements_per_sample = functools.reduce(operator.mul, size[1:], 1)
if elements_per_sample > 30:
return compute_mean_std(tensor)
else:
return subsample_tensor(tensor)
def subsample_tensor(tensor):
num_elems = tensor.size(0)
num_samples = 20
if num_elems <= num_samples:
return tensor
ith_index = num_elems // num_samples
return tensor[ith_index - 1::ith_index]
def compute_mean_std(tensor):
# can't compute mean of integral tensor
tensor = tensor.to(torch.double)
mean = torch.mean(tensor)
std = torch.std(tensor)
return {"mean": mean, "std": std}
output = map_nested_tensor_object(out, tensor_map_fn=compact)
prec = 0.01
try:
# We first try to assert the entire output if possible. This is not
# only the best way to assert results but also handles the cases
# where we need to create a new expected result.
_assert_expected(output, model_name, prec=prec)
except AssertionError:
# Unfortunately detection models are flaky due to the unstable sort
# in NMS. If matching across all outputs fails, use the same approach
# as in NMSTester.test_nms_cuda to see if this is caused by duplicate
# scores.
expected_file = _get_expected_file(model_name)
expected = torch.load(expected_file)
torch.testing.assert_close(output[0]["scores"], expected[0]["scores"], rtol=prec, atol=prec,
check_device=False, check_dtype=False)
# Note: Fmassa proposed turning off NMS by adapting the threshold
# and then using the Hungarian algorithm as in DETR to find the
# best match between output and expected boxes and eliminate some
# of the flakiness. Worth exploring.
return False # Partial validation performed
return True # Full validation performed
full_validation = check_out(out)
_check_jit_scriptable(model, ([x],), unwrapper=script_model_unwrapper.get(model_name, None))
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast():
out = model(model_input)
# See autocast_flaky_numerics comment at top of file.
if model_name not in autocast_flaky_numerics:
full_validation &= check_out(out)
if not full_validation:
msg = "The output of {} could only be partially validated. " \
"This is likely due to unit-test flakiness, but you may " \
"want to do additional manual checks if you made " \
"significant changes to the codebase.".format(test_detection_model.__name__)
warnings.warn(msg, RuntimeWarning)
pytest.skip(msg)
@cpu_only
@pytest.mark.parametrize('model_name', get_available_detection_models()) @pytest.mark.parametrize('model_name', get_available_detection_models())
def test_detection_model_validation(model_name): def test_detection_model_validation(model_name):
ModelTester()._test_detection_model_validation(model_name) set_rng_seed(0)
model = models.detection.__dict__[model_name](num_classes=50, pretrained_backbone=False)
input_shape = (3, 300, 300)
x = [torch.rand(input_shape)]
# validate that targets are present in training
with pytest.raises(ValueError):
model(x)
# validate type
targets = [{'boxes': 0.}]
with pytest.raises(ValueError):
model(x, targets=targets)
# validate boxes shape
for boxes in (torch.rand((4,)), torch.rand((1, 5))):
targets = [{'boxes': boxes}]
with pytest.raises(ValueError):
model(x, targets=targets)
# validate that no degenerate boxes are present
boxes = torch.tensor([[1, 3, 1, 4], [2, 4, 3, 4]])
targets = [{'boxes': boxes}]
with pytest.raises(ValueError):
model(x, targets=targets)
@pytest.mark.parametrize('model_name', get_available_video_models()) @pytest.mark.parametrize('model_name', get_available_video_models())
@pytest.mark.parametrize('dev', _devs) @pytest.mark.parametrize('dev', cpu_and_gpu())
def test_video_model(model_name, dev): def test_video_model(model_name, dev):
ModelTester()._test_video_model(model_name, dev) # the default input shape is
# bs * num_channels * clip_len * h *w
input_shape = (1, 3, 4, 112, 112)
# test both basicblock and Bottleneck
model = models.video.__dict__[model_name](num_classes=50)
model.eval().to(device=dev)
# RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
x = torch.rand(input_shape).to(device=dev)
out = model(x)
_check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(model_name, None))
assert out.shape[-1] == 50
if dev == torch.device("cuda"):
with torch.cuda.amp.autocast():
out = model(x)
assert out.shape[-1] == 50
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment