test_models.py 7.05 KB
Newer Older
1
2
from collections import OrderedDict
from itertools import product
3
4
5
import torch
from torchvision import models
import unittest
eellison's avatar
eellison committed
6
import traceback
7
8


9
def get_available_classification_models():
10
    # TODO add a registration mechanism to torchvision.models
11
12
13
14
15
16
    return [k for k, v in models.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]


def get_available_segmentation_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.segmentation.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]
17
18


19
20
21
22
23
def get_available_detection_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.detection.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]


24
25
26
27
28
def get_available_video_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.video.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]


29
30
31
32
33
34
35
36
37
38
39
40
41
42
torchub_models = [
    "deeplabv3_resnet101",
    "mobilenet_v2",
    "resnext50_32x4d",
    "fcn_resnet101",
    "googlenet",
    "densenet121",
    "resnet18",
    "alexnet",
    "shufflenet_v2_x1_0",
    "squeezenet1_0",
    "vgg11",
    "inception_v3",
]
43
44


45
class Tester(unittest.TestCase):
46
47
48
49
    def check_script(self, model, name):
        if name not in torchub_models:
            return
        scriptable = True
eellison's avatar
eellison committed
50
        msg = ""
51
52
        try:
            torch.jit.script(model)
eellison's avatar
eellison committed
53
54
        except Exception as e:
            tb = traceback.format_exc()
55
            scriptable = False
eellison's avatar
eellison committed
56
            msg = str(e) + str(tb)
57
        self.assertTrue(scriptable, msg)
58

59
    def _test_classification_model(self, name, input_shape):
60
61
        # passing num_class equal to a number other than 1000 helps in making the test
        # more enforcing in nature
62
        model = models.__dict__[name](num_classes=50)
63
        self.check_script(model, name)
64
65
66
        model.eval()
        x = torch.rand(input_shape)
        out = model(x)
67
        self.assertEqual(out.shape[-1], 50)
68

69
70
71
72
    def _test_segmentation_model(self, name):
        # passing num_class equal to a number other than 1000 helps in making the test
        # more enforcing in nature
        model = models.segmentation.__dict__[name](num_classes=50, pretrained_backbone=False)
73
        self.check_script(model, name)
74
75
76
77
78
79
        model.eval()
        input_shape = (1, 3, 300, 300)
        x = torch.rand(input_shape)
        out = model(x)
        self.assertEqual(tuple(out["out"].shape), (1, 50, 300, 300))

80
81
    def _test_detection_model(self, name):
        model = models.detection.__dict__[name](num_classes=50, pretrained_backbone=False)
82
        self.check_script(model, name)
83
84
85
        model.eval()
        input_shape = (3, 300, 300)
        x = torch.rand(input_shape)
86
87
88
        model_input = [x]
        out = model(model_input)
        self.assertIs(model_input[0], x)
89
90
91
92
93
        self.assertEqual(len(out), 1)
        self.assertTrue("boxes" in out[0])
        self.assertTrue("scores" in out[0])
        self.assertTrue("labels" in out[0])

94
95
96
    def _test_video_model(self, name):
        # the default input shape is
        # bs * num_channels * clip_len * h *w
97
        input_shape = (1, 3, 4, 112, 112)
98
99
        # test both basicblock and Bottleneck
        model = models.video.__dict__[name](num_classes=50)
100
        self.check_script(model, name)
101
102
103
104
        x = torch.rand(input_shape)
        out = model(x)
        self.assertEqual(out.shape[-1], 50)

105
106
107
108
109
110
111
112
113
    def _make_sliced_model(self, model, stop_layer):
        layers = OrderedDict()
        for name, layer in model.named_children():
            layers[name] = layer
            if name == stop_layer:
                break
        new_model = torch.nn.Sequential(layers)
        return new_model

114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    def test_memory_efficient_densenet(self):
        input_shape = (1, 3, 300, 300)
        x = torch.rand(input_shape)

        for name in ['densenet121', 'densenet169', 'densenet201', 'densenet161']:
            model1 = models.__dict__[name](num_classes=50, memory_efficient=True)
            params = model1.state_dict()
            model1.eval()
            out1 = model1(x)
            out1.sum().backward()

            model2 = models.__dict__[name](num_classes=50, memory_efficient=False)
            model2.load_state_dict(params)
            model2.eval()
            out2 = model2(x)

            max_diff = (out1 - out2).abs().max()

            self.assertTrue(max_diff < 1e-5)

134
135
136
137
138
139
140
141
142
143
144
    def test_resnet_dilation(self):
        # TODO improve tests to also check that each layer has the right dimensionality
        for i in product([False, True], [False, True], [False, True]):
            model = models.__dict__["resnet50"](replace_stride_with_dilation=i)
            model = self._make_sliced_model(model, stop_layer="layer4")
            model.eval()
            x = torch.rand(1, 3, 224, 224)
            out = model(x)
            f = 2 ** sum(i)
            self.assertEqual(out.shape, (1, 2048, 7 * f, 7 * f))

145
146
147
148
149
150
151
    def test_mobilenetv2_residual_setting(self):
        model = models.__dict__["mobilenet_v2"](inverted_residual_setting=[[1, 16, 1, 1], [6, 24, 2, 2]])
        model.eval()
        x = torch.rand(1, 3, 224, 224)
        out = model(x)
        self.assertEqual(out.shape[-1], 1000)

152
153
154
155
156
157
158
159
160
161
162
163
164
165
    def test_fasterrcnn_double(self):
        model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
        model.double()
        model.eval()
        input_shape = (3, 300, 300)
        x = torch.rand(input_shape, dtype=torch.float64)
        model_input = [x]
        out = model(model_input)
        self.assertIs(model_input[0], x)
        self.assertEqual(len(out), 1)
        self.assertTrue("boxes" in out[0])
        self.assertTrue("scores" in out[0])
        self.assertTrue("labels" in out[0])

166

167
for model_name in get_available_classification_models():
168
169
170
171
172
173
    # for-loop bodies don't define scopes, so we have to save the variables
    # we want to close over in some way
    def do_test(self, model_name=model_name):
        input_shape = (1, 3, 224, 224)
        if model_name in ['inception_v3']:
            input_shape = (1, 3, 299, 299)
174
175
176
177
178
179
180
181
182
183
        self._test_classification_model(model_name, input_shape)

    setattr(Tester, "test_" + model_name, do_test)


for model_name in get_available_segmentation_models():
    # for-loop bodies don't define scopes, so we have to save the variables
    # we want to close over in some way
    def do_test(self, model_name=model_name):
        self._test_segmentation_model(model_name)
184
185
186
187

    setattr(Tester, "test_" + model_name, do_test)


188
189
190
191
192
193
194
195
for model_name in get_available_detection_models():
    # for-loop bodies don't define scopes, so we have to save the variables
    # we want to close over in some way
    def do_test(self, model_name=model_name):
        self._test_detection_model(model_name)

    setattr(Tester, "test_" + model_name, do_test)

196

197
198
199
200
201
202
for model_name in get_available_video_models():

    def do_test(self, model_name=model_name):
        self._test_video_model(model_name)

    setattr(Tester, "test_" + model_name, do_test)
203

204
205
if __name__ == '__main__':
    unittest.main()