test_pytorch_examples.py 22 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2018 HuggingFace Inc..
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Aymeric Augustin's avatar
Aymeric Augustin committed
15

16

17
import json
18
import logging
19
import os
Aymeric Augustin's avatar
Aymeric Augustin committed
20
import sys
Aymeric Augustin's avatar
Aymeric Augustin committed
21
from unittest.mock import patch
Aymeric Augustin's avatar
Aymeric Augustin committed
22

23
from transformers import ViTMAEForPreTraining, Wav2Vec2ForPreTraining
24
25
26
27
28
29
30
31
from transformers.testing_utils import (
    CaptureLogger,
    TestCasePlus,
    backend_device_count,
    is_torch_fp16_available_on_device,
    slow,
    torch_device,
)
32

33
34
35

SRC_DIRS = [
    os.path.join(os.path.dirname(__file__), dirname)
36
37
38
39
40
    for dirname in [
        "text-generation",
        "text-classification",
        "token-classification",
        "language-modeling",
41
        "multiple-choice",
42
        "question-answering",
Sylvain Gugger's avatar
Sylvain Gugger committed
43
44
        "summarization",
        "translation",
45
        "image-classification",
46
        "speech-recognition",
47
        "audio-classification",
48
        "speech-pretraining",
49
        "image-pretraining",
50
        "semantic-segmentation",
51
        "object-detection",
52
    ]
53
54
55
56
57
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
58
    import run_audio_classification
Sylvain Gugger's avatar
Sylvain Gugger committed
59
    import run_clm
60
61
    import run_generation
    import run_glue
62
    import run_image_classification
63
    import run_mae
64
    import run_mlm
65
    import run_ner
66
    import run_object_detection
Sylvain Gugger's avatar
Sylvain Gugger committed
67
    import run_qa as run_squad
68
    import run_semantic_segmentation
69
    import run_seq2seq_qa as run_squad_seq2seq
70
    import run_speech_recognition_ctc
71
    import run_speech_recognition_ctc_adapter
72
    import run_speech_recognition_seq2seq
73
    import run_summarization
74
    import run_swag
75
    import run_translation
76
    import run_wav2vec2_pretraining_no_trainer
Aymeric Augustin's avatar
Aymeric Augustin committed
77

78

79
80
81
logging.basicConfig(level=logging.DEBUG)

logger = logging.getLogger()
82

83

84
85
86
87
88
89
90
91
92
93
94
def get_results(output_dir):
    results = {}
    path = os.path.join(output_dir, "all_results.json")
    if os.path.exists(path):
        with open(path, "r") as f:
            results = json.load(f)
    else:
        raise ValueError(f"can't find {path}")
    return results


95
96
97
98
stream_handler = logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)


99
class ExamplesTests(TestCasePlus):
100
    def test_run_glue(self):
101
102
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
103
            run_glue.py
104
            --model_name_or_path distilbert/distilbert-base-uncased
105
106
            --output_dir {tmp_dir}
            --overwrite_output_dir
Sylvain Gugger's avatar
Sylvain Gugger committed
107
108
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
109
110
            --do_train
            --do_eval
111
112
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
113
114
115
116
117
            --learning_rate=1e-4
            --max_steps=10
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
118
            """.split()
119

120
        if is_torch_fp16_available_on_device(torch_device):
121
            testargs.append("--fp16")
122

123
        with patch.object(sys, "argv", testargs):
124
125
            run_glue.main()
            result = get_results(tmp_dir)
126
            self.assertGreaterEqual(result["eval_accuracy"], 0.75)
127

Sylvain Gugger's avatar
Sylvain Gugger committed
128
129
130
131
    def test_run_clm(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_clm.py
132
            --model_name_or_path distilbert/distilgpt2
Sylvain Gugger's avatar
Sylvain Gugger committed
133
134
135
136
137
138
139
140
141
142
143
144
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --block_size 128
            --per_device_train_batch_size 5
            --per_device_eval_batch_size 5
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            """.split()

145
        if backend_device_count(torch_device) > 1:
Sylvain Gugger's avatar
Sylvain Gugger committed
146
147
148
            # Skipping because there are not enough batches to train the model + would need a drop_last to work.
            return

149
150
        if torch_device == "cpu":
            testargs.append("--use_cpu")
Sylvain Gugger's avatar
Sylvain Gugger committed
151
152

        with patch.object(sys, "argv", testargs):
153
154
            run_clm.main()
            result = get_results(tmp_dir)
Sylvain Gugger's avatar
Sylvain Gugger committed
155
156
            self.assertLess(result["perplexity"], 100)

157
158
159
160
161
162
163
164
    def test_run_clm_config_overrides(self):
        # test that config_overrides works, despite the misleading dumps of default un-updated
        # config via tokenizer

        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_clm.py
            --model_type gpt2
165
            --tokenizer_name openai-community/gpt2
166
167
168
169
170
            --train_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --config_overrides n_embd=10,n_head=2
            """.split()

171
172
        if torch_device == "cpu":
            testargs.append("--use_cpu")
173
174
175
176
177
178
179
180
181

        logger = run_clm.logger
        with patch.object(sys, "argv", testargs):
            with CaptureLogger(logger) as cl:
                run_clm.main()

        self.assertIn('"n_embd": 10', cl.out)
        self.assertIn('"n_head": 2', cl.out)

182
    def test_run_mlm(self):
183
184
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
185
            run_mlm.py
186
            --model_name_or_path distilbert/distilroberta-base
187
188
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
189
            --output_dir {tmp_dir}
Julien Chaumond's avatar
Julien Chaumond committed
190
191
192
            --overwrite_output_dir
            --do_train
            --do_eval
193
            --prediction_loss_only
Julien Chaumond's avatar
Julien Chaumond committed
194
            --num_train_epochs=1
195
        """.split()
196

197
198
        if torch_device == "cpu":
            testargs.append("--use_cpu")
199

Julien Chaumond's avatar
Julien Chaumond committed
200
        with patch.object(sys, "argv", testargs):
201
202
            run_mlm.main()
            result = get_results(tmp_dir)
203
            self.assertLess(result["perplexity"], 42)
Julien Chaumond's avatar
Julien Chaumond committed
204

205
    def test_run_ner(self):
206
        # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
207
        epochs = 7 if backend_device_count(torch_device) > 1 else 2
208

209
210
211
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_ner.py
212
            --model_name_or_path google-bert/bert-base-uncased
213
214
215
216
217
218
219
220
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --warmup_steps=2
            --learning_rate=2e-4
Sylvain Gugger's avatar
Sylvain Gugger committed
221
222
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
223
            --num_train_epochs={epochs}
224
            --seed 7
225
226
        """.split()

227
228
        if torch_device == "cpu":
            testargs.append("--use_cpu")
229
230

        with patch.object(sys, "argv", testargs):
231
232
            run_ner.main()
            result = get_results(tmp_dir)
233
            self.assertGreaterEqual(result["eval_accuracy"], 0.75)
234
235
            self.assertLess(result["eval_loss"], 0.5)

236
    def test_run_squad(self):
237
238
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
Russell Klopfer's avatar
Russell Klopfer committed
239
            run_qa.py
240
            --model_name_or_path google-bert/bert-base-uncased
Sylvain Gugger's avatar
Sylvain Gugger committed
241
242
243
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
244
245
            --output_dir {tmp_dir}
            --overwrite_output_dir
246
247
248
249
250
            --max_steps=10
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
Sylvain Gugger's avatar
Sylvain Gugger committed
251
252
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
253
254
        """.split()

255
        with patch.object(sys, "argv", testargs):
256
257
            run_squad.main()
            result = get_results(tmp_dir)
Russell Klopfer's avatar
Russell Klopfer committed
258
259
            self.assertGreaterEqual(result["eval_f1"], 30)
            self.assertGreaterEqual(result["eval_exact"], 30)
260

261
262
263
264
    def test_run_squad_seq2seq(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_seq2seq_qa.py
265
            --model_name_or_path google-t5/t5-small
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
            --context_column context
            --question_column question
            --answer_column answers
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=10
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --predict_with_generate
        """.split()

        with patch.object(sys, "argv", testargs):
            run_squad_seq2seq.main()
            result = get_results(tmp_dir)
287
288
            self.assertGreaterEqual(result["eval_f1"], 30)
            self.assertGreaterEqual(result["eval_exact"], 30)
289

290
291
292
293
    def test_run_swag(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_swag.py
294
            --model_name_or_path google-bert/bert-base-uncased
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=20
            --warmup_steps=2
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        """.split()

        with patch.object(sys, "argv", testargs):
309
310
            run_swag.main()
            result = get_results(tmp_dir)
311
312
            self.assertGreaterEqual(result["eval_accuracy"], 0.8)

313
    def test_generation(self):
314
        testargs = ["run_generation.py", "--prompt=Hello", "--length=10", "--seed=42"]
315

316
        if is_torch_fp16_available_on_device(torch_device):
317
318
319
320
321
322
            testargs.append("--fp16")

        model_type, model_name = (
            "--model_type=gpt2",
            "--model_name_or_path=sshleifer/tiny-gpt2",
        )
323
        with patch.object(sys, "argv", testargs + [model_type, model_name]):
324
            result = run_generation.main()
325
            self.assertGreaterEqual(len(result[0]), 10)
326
327

    @slow
328
    def test_run_summarization(self):
329
330
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
331
            run_summarization.py
332
            --model_name_or_path google-t5/t5-small
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=50
            --warmup_steps=8
            --do_train
            --do_eval
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --predict_with_generate
        """.split()

        with patch.object(sys, "argv", testargs):
348
            run_summarization.main()
349
            result = get_results(tmp_dir)
350
351
352
353
354
355
            self.assertGreaterEqual(result["eval_rouge1"], 10)
            self.assertGreaterEqual(result["eval_rouge2"], 2)
            self.assertGreaterEqual(result["eval_rougeL"], 7)
            self.assertGreaterEqual(result["eval_rougeLsum"], 7)

    @slow
356
    def test_run_translation(self):
357
358
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
359
            run_translation.py
360
            --model_name_or_path sshleifer/student_marian_en_ro_6_1
361
362
            --source_lang en
            --target_lang ro
363
364
365
366
367
368
369
370
371
372
373
374
375
376
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_steps=50
            --warmup_steps=8
            --do_train
            --do_eval
            --learning_rate=3e-3
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --predict_with_generate
            --source_lang en_XX
            --target_lang ro_RO
377
            --max_source_length 512
378
379
380
        """.split()

        with patch.object(sys, "argv", testargs):
381
            run_translation.main()
382
            result = get_results(tmp_dir)
383
            self.assertGreaterEqual(result["eval_bleu"], 30)
384
385
386
387
388
389
390

    def test_run_image_classification(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_image_classification.py
            --output_dir {tmp_dir}
            --model_name_or_path google/vit-base-patch16-224-in21k
391
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
392
393
            --do_train
            --do_eval
394
            --learning_rate 1e-4
395
396
397
398
399
400
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --remove_unused_columns False
            --overwrite_output_dir True
            --dataloader_num_workers 16
            --metric_for_best_model accuracy
401
            --max_steps 10
402
            --train_val_split 0.1
403
            --seed 42
404
            --label_column_name labels
405
406
        """.split()

407
        if is_torch_fp16_available_on_device(torch_device):
408
409
410
411
412
413
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_image_classification.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_accuracy"], 0.8)
414
415
416
417
418
419
420

    def test_run_speech_recognition_ctc(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_speech_recognition_ctc.py
            --output_dir {tmp_dir}
            --model_name_or_path hf-internal-testing/tiny-random-wav2vec2
Patrick von Platen's avatar
Patrick von Platen committed
421
            --dataset_name hf-internal-testing/librispeech_asr_dummy
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
            --dataset_config_name clean
            --train_split_name validation
            --eval_split_name validation
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --remove_unused_columns False
            --overwrite_output_dir True
            --preprocessing_num_workers 16
            --max_steps 10
            --seed 42
        """.split()

437
        if is_torch_fp16_available_on_device(torch_device):
438
439
440
441
442
443
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_speech_recognition_ctc.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_loss"], result["train_loss"])
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467

    def test_run_speech_recognition_ctc_adapter(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_speech_recognition_ctc_adapter.py
            --output_dir {tmp_dir}
            --model_name_or_path hf-internal-testing/tiny-random-wav2vec2
            --dataset_name hf-internal-testing/librispeech_asr_dummy
            --dataset_config_name clean
            --train_split_name validation
            --eval_split_name validation
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --remove_unused_columns False
            --overwrite_output_dir True
            --preprocessing_num_workers 16
            --max_steps 10
            --target_language tur
            --seed 42
        """.split()

468
        if is_torch_fp16_available_on_device(torch_device):
469
470
471
472
473
474
475
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_speech_recognition_ctc_adapter.main()
            result = get_results(tmp_dir)
            self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "./adapter.tur.safetensors")))
            self.assertLess(result["eval_loss"], result["train_loss"])
476

477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
    def test_run_speech_recognition_seq2seq(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_speech_recognition_seq2seq.py
            --output_dir {tmp_dir}
            --model_name_or_path hf-internal-testing/tiny-random-speech-encoder-decoder
            --dataset_name hf-internal-testing/librispeech_asr_dummy
            --dataset_config_name clean
            --train_split_name validation
            --eval_split_name validation
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 4
            --remove_unused_columns False
            --overwrite_output_dir True
            --preprocessing_num_workers 16
            --max_steps 10
            --seed 42
        """.split()

499
        if is_torch_fp16_available_on_device(torch_device):
500
501
502
503
504
505
506
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_speech_recognition_seq2seq.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_loss"], result["train_loss"])

507
508
509
510
511
512
513
514
515
516
    def test_run_audio_classification(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_audio_classification.py
            --output_dir {tmp_dir}
            --model_name_or_path hf-internal-testing/tiny-random-wav2vec2
            --dataset_name anton-l/superb_demo
            --dataset_config_name ks
            --train_split_name test
            --eval_split_name test
517
            --audio_column_name audio
518
519
520
521
522
523
524
525
526
527
528
529
530
            --label_column_name label
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --remove_unused_columns False
            --overwrite_output_dir True
            --num_train_epochs 10
            --max_steps 50
            --seed 42
        """.split()

531
        if is_torch_fp16_available_on_device(torch_device):
532
533
534
535
536
537
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_audio_classification.main()
            result = get_results(tmp_dir)
            self.assertLess(result["eval_loss"], result["train_loss"])
538
539
540
541
542
543
544

    def test_run_wav2vec2_pretraining(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_wav2vec2_pretraining_no_trainer.py
            --output_dir {tmp_dir}
            --model_name_or_path hf-internal-testing/tiny-random-wav2vec2
Patrick von Platen's avatar
Patrick von Platen committed
545
            --dataset_name hf-internal-testing/librispeech_asr_dummy
546
547
548
            --dataset_config_names clean
            --dataset_split_names validation
            --learning_rate 1e-4
549
550
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
551
            --preprocessing_num_workers 16
552
            --max_train_steps 2
553
554
555
556
557
558
559
560
            --validation_split_percentage 5
            --seed 42
        """.split()

        with patch.object(sys, "argv", testargs):
            run_wav2vec2_pretraining_no_trainer.main()
            model = Wav2Vec2ForPreTraining.from_pretrained(tmp_dir)
            self.assertIsNotNone(model)
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581

    def test_run_vit_mae_pretraining(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_mae.py
            --output_dir {tmp_dir}
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --do_train
            --do_eval
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --remove_unused_columns False
            --overwrite_output_dir True
            --dataloader_num_workers 16
            --metric_for_best_model accuracy
            --max_steps 10
            --train_val_split 0.1
            --seed 42
        """.split()

582
        if is_torch_fp16_available_on_device(torch_device):
583
584
585
586
587
588
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_mae.main()
            model = ViTMAEForPreTraining.from_pretrained(tmp_dir)
            self.assertIsNotNone(model)
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606

    def test_run_semantic_segmentation(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_semantic_segmentation.py
            --output_dir {tmp_dir}
            --dataset_name huggingface/semantic-segmentation-test-sample
            --do_train
            --do_eval
            --remove_unused_columns False
            --overwrite_output_dir True
            --max_steps 10
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --seed 32
        """.split()

607
        if is_torch_fp16_available_on_device(torch_device):
608
609
610
611
612
613
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_semantic_segmentation.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["eval_overall_accuracy"], 0.1)
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641

    @patch.dict(os.environ, {"WANDB_DISABLED": "true"})
    def test_run_object_detection(self):
        tmp_dir = self.get_auto_remove_tmp_dir()
        testargs = f"""
            run_object_detection.py
            --model_name_or_path qubvel-hf/detr-resnet-50-finetuned-10k-cppe5
            --output_dir {tmp_dir}
            --dataset_name qubvel-hf/cppe-5-sample
            --do_train
            --do_eval
            --remove_unused_columns False
            --overwrite_output_dir True
            --eval_do_concat_batches False
            --max_steps 10
            --learning_rate=1e-6
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --seed 32
        """.split()

        if is_torch_fp16_available_on_device(torch_device):
            testargs.append("--fp16")

        with patch.object(sys, "argv", testargs):
            run_object_detection.main()
            result = get_results(tmp_dir)
            self.assertGreaterEqual(result["test_map"], 0.1)