test_models.py 24.3 KB
Newer Older
1
2
import os
import io
3
import sys
4
5
from common_utils import TestCase, map_nested_tensor_object, freeze_rng_state, set_rng_seed
from _utils_internal import get_relative_path
6
7
from collections import OrderedDict
from itertools import product
8
9
import functools
import operator
10
import torch
11
import torch.nn as nn
12
13
from torchvision import models
import unittest
14
import warnings
15

16
17
import pytest

eellison's avatar
eellison committed
18

19
20
21
ACCEPT = os.getenv('EXPECTTEST_ACCEPT', '0') == '1'


22
23
24
def get_available_classification_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]
25
26
27
28
29


def get_available_segmentation_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.segmentation.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]
30
31


32
33
34
35
36
def get_available_detection_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.detection.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]


37
38
39
40
41
def get_available_video_models():
    # TODO add a registration mechanism to torchvision.models
    return [k for k, v in models.video.__dict__.items() if callable(v) and k[0].lower() == k[0] and k[0] != "_"]


42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def _get_expected_file(name=None):
    # Determine expected file based on environment
    expected_file_base = get_relative_path(os.path.realpath(__file__), "expect")

    # Note: for legacy reasons, the reference file names all had "ModelTest.test_" in their names
    # We hardcode it here to avoid having to re-generate the reference files
    expected_file = expected_file = os.path.join(expected_file_base, 'ModelTester.test_' + name)
    expected_file += "_expect.pkl"

    if not ACCEPT and not os.path.exists(expected_file):
        raise RuntimeError(
            f"No expect file exists for {os.path.basename(expected_file)} in {expected_file}; "
            "to accept the current output, re-run the failing test after setting the EXPECTTEST_ACCEPT "
            "env variable. For example: EXPECTTEST_ACCEPT=1 pytest test/test_models.py -k alexnet"
        )

    return expected_file


def _assert_expected(output, name, prec):
    """Test that a python value matches the recorded contents of a file
    based on a "check" name. The value must be
    pickable with `torch.save`. This file
    is placed in the 'expect' directory in the same directory
    as the test script. You can automatically update the recorded test
    output using an EXPECTTEST_ACCEPT=1 env variable.
    """
    expected_file = _get_expected_file(name)

    if ACCEPT:
        filename = {os.path.basename(expected_file)}
        print("Accepting updated output for {}:\n\n{}".format(filename, output))
        torch.save(output, expected_file)
        MAX_PICKLE_SIZE = 50 * 1000  # 50 KB
        binary_size = os.path.getsize(expected_file)
        if binary_size > MAX_PICKLE_SIZE:
            raise RuntimeError("The output for {}, is larger than 50kb".format(filename))
    else:
        expected = torch.load(expected_file)
        rtol = atol = prec
        torch.testing.assert_close(output, expected, rtol=rtol, atol=atol, check_dtype=False)


def _check_jit_scriptable(nn_module, args, unwrapper=None, skip=False):
    """Check that a nn.Module's results in TorchScript match eager and that it can be exported"""

    def assert_export_import_module(m, args):
        """Check that the results of a model are the same after saving and loading"""
        def get_export_import_copy(m):
            """Save and load a TorchScript model"""
            buffer = io.BytesIO()
            torch.jit.save(m, buffer)
            buffer.seek(0)
            imported = torch.jit.load(buffer)
            return imported

        m_import = get_export_import_copy(m)
        with freeze_rng_state():
            results = m(*args)
        with freeze_rng_state():
            results_from_imported = m_import(*args)
        tol = 3e-4
        try:
            torch.testing.assert_close(results, results_from_imported, atol=tol, rtol=tol)
        except pytest.UsageError:
            # custom check for the models that return named tuples:
            # we compare field by field while ignoring None as assert_close can't handle None
            for a, b in zip(results, results_from_imported):
                if a is not None:
                    torch.testing.assert_close(a, b, atol=tol, rtol=tol)

    TEST_WITH_SLOW = os.getenv('PYTORCH_TEST_WITH_SLOW', '0') == '1'
    if not TEST_WITH_SLOW or skip:
        # TorchScript is not enabled, skip these tests
        msg = "The check_jit_scriptable test for {} was skipped. " \
              "This test checks if the module's results in TorchScript " \
              "match eager and that it can be exported. To run these " \
              "tests make sure you set the environment variable " \
              "PYTORCH_TEST_WITH_SLOW=1 and that the test is not " \
              "manually skipped.".format(nn_module.__class__.__name__)
        warnings.warn(msg, RuntimeWarning)
        return None

    sm = torch.jit.script(nn_module)

    with freeze_rng_state():
        eager_out = nn_module(*args)

    with freeze_rng_state():
        script_out = sm(*args)
        if unwrapper:
            script_out = unwrapper(script_out)

    torch.testing.assert_close(eager_out, script_out, atol=1e-4, rtol=1e-4)
    assert_export_import_module(sm, args)


139
140
141
# If 'unwrapper' is provided it will be called with the script model outputs
# before they are compared to the eager model outputs. This is useful if the
# model outputs are different between TorchScript / Eager mode
142
143
144
145
script_model_unwrapper = {
    'googlenet': lambda x: x.logits,
    'inception_v3': lambda x: x.logits,
    "fasterrcnn_resnet50_fpn": lambda x: x[1],
146
    "fasterrcnn_mobilenet_v3_large_fpn": lambda x: x[1],
147
    "fasterrcnn_mobilenet_v3_large_320_fpn": lambda x: x[1],
148
149
150
    "maskrcnn_resnet50_fpn": lambda x: x[1],
    "keypointrcnn_resnet50_fpn": lambda x: x[1],
    "retinanet_resnet50_fpn": lambda x: x[1],
151
    "ssd300_vgg16": lambda x: x[1],
152
    "ssdlite320_mobilenet_v3_large": lambda x: x[1],
153
}
154
155


156
157
158
159
160
161
162
163
164
165
166
167
168
169
# The following models exhibit flaky numerics under autocast in _test_*_model harnesses.
# This may be caused by the harness environment (e.g. num classes, input initialization
# via torch.rand), and does not prove autocast is unsuitable when training with real data
# (autocast has been used successfully with real data for some of these models).
# TODO:  investigate why autocast numerics are flaky in the harnesses.
#
# For the following models, _test_*_model harnesses skip numerical checks on outputs when
# trying autocast. However, they still try an autocasted forward pass, so they still ensure
# autocast coverage suffices to prevent dtype errors in each model.
autocast_flaky_numerics = (
    "inception_v3",
    "resnet101",
    "resnet152",
    "wide_resnet101_2",
170
171
    "deeplabv3_resnet50",
    "deeplabv3_resnet101",
172
    "deeplabv3_mobilenet_v3_large",
173
174
    "fcn_resnet50",
    "fcn_resnet101",
175
    "lraspp_mobilenet_v3_large",
176
    "maskrcnn_resnet50_fpn",
177
178
179
)


180
181
182
183
184
185
186
# The following contains configuration parameters for all models which are used by
# the _test_*_model methods.
_model_params = {
    'inception_v3': {
        'input_shape': (1, 3, 299, 299)
    },
    'retinanet_resnet50_fpn': {
187
        'num_classes': 20,
188
        'score_thresh': 0.01,
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
        'min_size': 224,
        'max_size': 224,
        'input_shape': (3, 224, 224),
    },
    'keypointrcnn_resnet50_fpn': {
        'num_classes': 2,
        'min_size': 224,
        'max_size': 224,
        'box_score_thresh': 0.15,
        'input_shape': (3, 224, 224),
    },
    'fasterrcnn_resnet50_fpn': {
        'num_classes': 20,
        'min_size': 224,
        'max_size': 224,
        'input_shape': (3, 224, 224),
    },
    'maskrcnn_resnet50_fpn': {
        'num_classes': 10,
        'min_size': 224,
        'max_size': 224,
        'input_shape': (3, 224, 224),
211
212
213
214
215
216
217
218
219
220
221
222
    },
    'fasterrcnn_mobilenet_v3_large_fpn': {
        'box_score_thresh': 0.02076,
    },
    'fasterrcnn_mobilenet_v3_large_320_fpn': {
        'box_score_thresh': 0.02076,
        'rpn_pre_nms_top_n_test': 1000,
        'rpn_post_nms_top_n_test': 1000,
    }
}


eellison's avatar
eellison committed
223
class ModelTester(TestCase):
224
    def _test_classification_model(self, name, dev):
225
        set_rng_seed(0)
226
227
228
229
230
231
232
233
        defaults = {
            'num_classes': 50,
            'input_shape': (1, 3, 224, 224),
        }
        kwargs = {**defaults, **_model_params.get(name, {})}
        input_shape = kwargs.pop('input_shape')

        model = models.__dict__[name](**kwargs)
234
235
236
        model.eval().to(device=dev)
        # RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
        x = torch.rand(input_shape).to(device=dev)
237
        out = model(x)
238
        _assert_expected(out.cpu(), name, prec=0.1)
239
        self.assertEqual(out.shape[-1], 50)
240
        _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
241

242
        if dev == torch.device("cuda"):
243
244
245
246
            with torch.cuda.amp.autocast():
                out = model(x)
                # See autocast_flaky_numerics comment at top of file.
                if name not in autocast_flaky_numerics:
247
                    _assert_expected(out.cpu(), name, prec=0.1)
248
249
250
                self.assertEqual(out.shape[-1], 50)

    def _test_segmentation_model(self, name, dev):
251
        set_rng_seed(0)
252
253
254
255
256
257
258
259
260
        defaults = {
            'num_classes': 10,
            'pretrained_backbone': False,
            'input_shape': (1, 3, 32, 32),
        }
        kwargs = {**defaults, **_model_params.get(name, {})}
        input_shape = kwargs.pop('input_shape')

        model = models.segmentation.__dict__[name](**kwargs)
261
262
263
        model.eval().to(device=dev)
        # RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
        x = torch.rand(input_shape).to(device=dev)
264
265
266
267
268
269
270
271
        out = model(x)["out"]

        def check_out(out):
            prec = 0.01
            try:
                # We first try to assert the entire output if possible. This is not
                # only the best way to assert results but also handles the cases
                # where we need to create a new expected result.
272
                _assert_expected(out.cpu(), name, prec=prec)
273
274
275
276
            except AssertionError:
                # Unfortunately some segmentation models are flaky with autocast
                # so instead of validating the probability scores, check that the class
                # predictions match.
277
                expected_file = _get_expected_file(name)
278
                expected = torch.load(expected_file)
279
                torch.testing.assert_close(out.argmax(dim=1), expected.argmax(dim=1), rtol=prec, atol=prec)
280
281
282
283
284
285
                return False  # Partial validation performed

            return True  # Full validation performed

        full_validation = check_out(out)

286
        _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
287

288
        if dev == torch.device("cuda"):
289
            with torch.cuda.amp.autocast():
290
291
292
293
294
295
296
297
298
299
300
301
                out = model(x)["out"]
                # See autocast_flaky_numerics comment at top of file.
                if name not in autocast_flaky_numerics:
                    full_validation &= check_out(out)

        if not full_validation:
            msg = "The output of {} could only be partially validated. " \
                  "This is likely due to unit-test flakiness, but you may " \
                  "want to do additional manual checks if you made " \
                  "significant changes to the codebase.".format(self._testMethodName)
            warnings.warn(msg, RuntimeWarning)
            raise unittest.SkipTest(msg)
302
303

    def _test_detection_model(self, name, dev):
eellison's avatar
eellison committed
304
        set_rng_seed(0)
305
306
307
308
309
310
311
312
313
        defaults = {
            'num_classes': 50,
            'pretrained_backbone': False,
            'input_shape': (3, 300, 300),
        }
        kwargs = {**defaults, **_model_params.get(name, {})}
        input_shape = kwargs.pop('input_shape')

        model = models.detection.__dict__[name](**kwargs)
314
315
316
        model.eval().to(device=dev)
        # RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
        x = torch.rand(input_shape).to(device=dev)
317
318
319
        model_input = [x]
        out = model(model_input)
        self.assertIs(model_input[0], x)
eellison's avatar
eellison committed
320

321
322
323
        def check_out(out):
            self.assertEqual(len(out), 1)

324
325
326
327
328
329
330
331
            def compact(tensor):
                size = tensor.size()
                elements_per_sample = functools.reduce(operator.mul, size[1:], 1)
                if elements_per_sample > 30:
                    return compute_mean_std(tensor)
                else:
                    return subsample_tensor(tensor)

332
            def subsample_tensor(tensor):
333
                num_elems = tensor.size(0)
334
335
336
337
338
                num_samples = 20
                if num_elems <= num_samples:
                    return tensor

                ith_index = num_elems // num_samples
339
                return tensor[ith_index - 1::ith_index]
340
341
342
343
344
345
346
347

            def compute_mean_std(tensor):
                # can't compute mean of integral tensor
                tensor = tensor.to(torch.double)
                mean = torch.mean(tensor)
                std = torch.std(tensor)
                return {"mean": mean, "std": std}

348
349
350
351
352
353
            output = map_nested_tensor_object(out, tensor_map_fn=compact)
            prec = 0.01
            try:
                # We first try to assert the entire output if possible. This is not
                # only the best way to assert results but also handles the cases
                # where we need to create a new expected result.
354
                _assert_expected(output, name, prec=prec)
355
356
357
358
359
            except AssertionError:
                # Unfortunately detection models are flaky due to the unstable sort
                # in NMS. If matching across all outputs fails, use the same approach
                # as in NMSTester.test_nms_cuda to see if this is caused by duplicate
                # scores.
360
                expected_file = _get_expected_file(name)
361
                expected = torch.load(expected_file)
362
363
                torch.testing.assert_close(output[0]["scores"], expected[0]["scores"], rtol=prec, atol=prec,
                                           check_device=False, check_dtype=False)
364
365
366
367
368
369
370
371
372
373

                # Note: Fmassa proposed turning off NMS by adapting the threshold
                # and then using the Hungarian algorithm as in DETR to find the
                # best match between output and expected boxes and eliminate some
                # of the flakiness. Worth exploring.
                return False  # Partial validation performed

            return True  # Full validation performed

        full_validation = check_out(out)
374
        _check_jit_scriptable(model, ([x],), unwrapper=script_model_unwrapper.get(name, None))
375

376
        if dev == torch.device("cuda"):
377
378
379
380
            with torch.cuda.amp.autocast():
                out = model(model_input)
                # See autocast_flaky_numerics comment at top of file.
                if name not in autocast_flaky_numerics:
381
382
383
384
385
386
387
388
389
                    full_validation &= check_out(out)

        if not full_validation:
            msg = "The output of {} could only be partially validated. " \
                  "This is likely due to unit-test flakiness, but you may " \
                  "want to do additional manual checks if you made " \
                  "significant changes to the codebase.".format(self._testMethodName)
            warnings.warn(msg, RuntimeWarning)
            raise unittest.SkipTest(msg)
390

391
392
393
    def _test_detection_model_validation(self, name):
        set_rng_seed(0)
        model = models.detection.__dict__[name](num_classes=50, pretrained_backbone=False)
394
        input_shape = (3, 300, 300)
395
396
397
398
399
400
401
402
403
404
405
406
407
408
        x = [torch.rand(input_shape)]

        # validate that targets are present in training
        self.assertRaises(ValueError, model, x)

        # validate type
        targets = [{'boxes': 0.}]
        self.assertRaises(ValueError, model, x, targets=targets)

        # validate boxes shape
        for boxes in (torch.rand((4,)), torch.rand((1, 5))):
            targets = [{'boxes': boxes}]
            self.assertRaises(ValueError, model, x, targets=targets)

409
410
411
412
413
        # validate that no degenerate boxes are present
        boxes = torch.tensor([[1, 3, 1, 4], [2, 4, 3, 4]])
        targets = [{'boxes': boxes}]
        self.assertRaises(ValueError, model, x, targets=targets)

414
    def _test_video_model(self, name, dev):
415
416
417
418
419
        # the default input shape is
        # bs * num_channels * clip_len * h *w
        input_shape = (1, 3, 4, 112, 112)
        # test both basicblock and Bottleneck
        model = models.video.__dict__[name](num_classes=50)
420
421
422
        model.eval().to(device=dev)
        # RNG always on CPU, to ensure x in cuda tests is bitwise identical to x in cpu tests
        x = torch.rand(input_shape).to(device=dev)
423
        out = model(x)
424
        _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
425
426
        self.assertEqual(out.shape[-1], 50)

427
        if dev == torch.device("cuda"):
428
429
430
431
            with torch.cuda.amp.autocast():
                out = model(x)
                self.assertEqual(out.shape[-1], 50)

432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
    def _make_sliced_model(self, model, stop_layer):
        layers = OrderedDict()
        for name, layer in model.named_children():
            layers[name] = layer
            if name == stop_layer:
                break
        new_model = torch.nn.Sequential(layers)
        return new_model

    def test_memory_efficient_densenet(self):
        input_shape = (1, 3, 300, 300)
        x = torch.rand(input_shape)

        for name in ['densenet121', 'densenet169', 'densenet201', 'densenet161']:
            model1 = models.__dict__[name](num_classes=50, memory_efficient=True)
            params = model1.state_dict()
448
            num_params = sum([x.numel() for x in model1.parameters()])
449
450
451
            model1.eval()
            out1 = model1(x)
            out1.sum().backward()
452
            num_grad = sum([x.grad.numel() for x in model1.parameters() if x.grad is not None])
453
454
455
456
457
458

            model2 = models.__dict__[name](num_classes=50, memory_efficient=False)
            model2.load_state_dict(params)
            model2.eval()
            out2 = model2(x)

459
            self.assertTrue(num_params == num_grad)
460
            torch.testing.assert_close(out1, out2, rtol=0.0, atol=1e-5)
461
462
463
464
465
466
467
468
469
470
471
472

    def test_resnet_dilation(self):
        # TODO improve tests to also check that each layer has the right dimensionality
        for i in product([False, True], [False, True], [False, True]):
            model = models.__dict__["resnet50"](replace_stride_with_dilation=i)
            model = self._make_sliced_model(model, stop_layer="layer4")
            model.eval()
            x = torch.rand(1, 3, 224, 224)
            out = model(x)
            f = 2 ** sum(i)
            self.assertEqual(out.shape, (1, 2048, 7 * f, 7 * f))

473
    def test_mobilenet_v2_residual_setting(self):
474
475
476
477
478
479
        model = models.__dict__["mobilenet_v2"](inverted_residual_setting=[[1, 16, 1, 1], [6, 24, 2, 2]])
        model.eval()
        x = torch.rand(1, 3, 224, 224)
        out = model(x)
        self.assertEqual(out.shape[-1], 1000)

480
481
482
483
    def test_mobilenet_norm_layer(self):
        for name in ["mobilenet_v2", "mobilenet_v3_large", "mobilenet_v3_small"]:
            model = models.__dict__[name]()
            self.assertTrue(any(isinstance(x, nn.BatchNorm2d) for x in model.modules()))
484

485
486
            def get_gn(num_channels):
                return nn.GroupNorm(32, num_channels)
487

488
489
490
            model = models.__dict__[name](norm_layer=get_gn)
            self.assertFalse(any(isinstance(x, nn.BatchNorm2d) for x in model.modules()))
            self.assertTrue(any(isinstance(x, nn.GroupNorm) for x in model.modules()))
491

492
    def test_inception_v3_eval(self):
493
494
495
496
497
        # replacement for models.inception_v3(pretrained=True) that does not download weights
        kwargs = {}
        kwargs['transform_input'] = True
        kwargs['aux_logits'] = True
        kwargs['init_weights'] = False
498
        name = "inception_v3"
499
500
501
        model = models.Inception3(**kwargs)
        model.aux_logits = False
        model.AuxLogits = None
502
503
        model = model.eval()
        x = torch.rand(1, 3, 299, 299)
504
        _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
505

506
507
508
509
510
511
512
513
514
515
516
517
518
519
    def test_fasterrcnn_double(self):
        model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
        model.double()
        model.eval()
        input_shape = (3, 300, 300)
        x = torch.rand(input_shape, dtype=torch.float64)
        model_input = [x]
        out = model(model_input)
        self.assertIs(model_input[0], x)
        self.assertEqual(len(out), 1)
        self.assertTrue("boxes" in out[0])
        self.assertTrue("scores" in out[0])
        self.assertTrue("labels" in out[0])

520
    def test_googlenet_eval(self):
521
522
523
524
525
        # replacement for models.googlenet(pretrained=True) that does not download weights
        kwargs = {}
        kwargs['transform_input'] = True
        kwargs['aux_logits'] = True
        kwargs['init_weights'] = False
526
        name = "googlenet"
527
528
529
530
        model = models.GoogLeNet(**kwargs)
        model.aux_logits = False
        model.aux1 = None
        model.aux2 = None
531
532
        model = model.eval()
        x = torch.rand(1, 3, 224, 224)
533
        _check_jit_scriptable(model, (x,), unwrapper=script_model_unwrapper.get(name, None))
534

535
536
    @unittest.skipIf(not torch.cuda.is_available(), 'needs GPU')
    def test_fasterrcnn_switch_devices(self):
537
538
539
540
541
542
        def checkOut(out):
            self.assertEqual(len(out), 1)
            self.assertTrue("boxes" in out[0])
            self.assertTrue("scores" in out[0])
            self.assertTrue("labels" in out[0])

543
544
545
546
547
548
549
550
        model = models.detection.fasterrcnn_resnet50_fpn(num_classes=50, pretrained_backbone=False)
        model.cuda()
        model.eval()
        input_shape = (3, 300, 300)
        x = torch.rand(input_shape, device='cuda')
        model_input = [x]
        out = model(model_input)
        self.assertIs(model_input[0], x)
551
552
553
554
555
556
557
558

        checkOut(out)

        with torch.cuda.amp.autocast():
            out = model(model_input)

        checkOut(out)

559
560
561
562
        # now switch to cpu and make sure it works
        model.cpu()
        x = x.cpu()
        out_cpu = model([x])
563
564

        checkOut(out_cpu)
565

566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
    def test_generalizedrcnn_transform_repr(self):

        min_size, max_size = 224, 299
        image_mean = [0.485, 0.456, 0.406]
        image_std = [0.229, 0.224, 0.225]

        t = models.detection.transform.GeneralizedRCNNTransform(min_size=min_size,
                                                                max_size=max_size,
                                                                image_mean=image_mean,
                                                                image_std=image_std)

        # Check integrity of object __repr__ attribute
        expected_string = 'GeneralizedRCNNTransform('
        _indent = '\n    '
        expected_string += '{0}Normalize(mean={1}, std={2})'.format(_indent, image_mean, image_std)
        expected_string += '{0}Resize(min_size=({1},), max_size={2}, '.format(_indent, min_size, max_size)
        expected_string += "mode='bilinear')\n)"
        self.assertEqual(t.__repr__(), expected_string)

585

586
_devs = [torch.device("cpu"), torch.device("cuda")] if torch.cuda.is_available() else [torch.device("cpu")]
587
588


589
590
591
@pytest.mark.parametrize('model_name', get_available_classification_models())
@pytest.mark.parametrize('dev', _devs)
def test_classification_model(model_name, dev):
592
    ModelTester()._test_classification_model(model_name, dev)
593
594


595
596
597
598
@pytest.mark.parametrize('model_name', get_available_segmentation_models())
@pytest.mark.parametrize('dev', _devs)
def test_segmentation_model(model_name, dev):
    ModelTester()._test_segmentation_model(model_name, dev)
599

600

601
602
603
604
@pytest.mark.parametrize('model_name', get_available_detection_models())
@pytest.mark.parametrize('dev', _devs)
def test_detection_model(model_name, dev):
    ModelTester()._test_detection_model(model_name, dev)
605

606

607
608
609
@pytest.mark.parametrize('model_name', get_available_detection_models())
def test_detection_model_validation(model_name):
    ModelTester()._test_detection_model_validation(model_name)
610

611

612
613
614
615
@pytest.mark.parametrize('model_name', get_available_video_models())
@pytest.mark.parametrize('dev', _devs)
def test_video_model(model_name, dev):
    ModelTester()._test_video_model(model_name, dev)
616

617

618
if __name__ == '__main__':
619
    pytest.main([__file__])