mark `test_initialization` as flaky in 2 model tests (#27906)

fix
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/models/dinov2/test_modeling_dinov2.py

@@ -19,6 +19,7 @@ import unittest
 
 from transformers import Dinov2Config
 from transformers.testing_utils import (
+    is_flaky,
     require_torch,
     require_vision,
     slow,
@@ -230,6 +231,10 @@ class Dinov2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
         self.model_tester = Dinov2ModelTester(self)
         self.config_tester = ConfigTester(self, config_class=Dinov2Config, has_text_modality=False, hidden_size=37)
 
+    @is_flaky(max_attempts=3, description="`torch.nn.init.trunc_normal_` is flaky.")
+    def test_initialization(self):
+        super().test_initialization()
+
     def test_config(self):
         self.config_tester.run_common_tests()
 

tests/models/vitdet/test_modeling_vitdet.py

@@ -18,7 +18,7 @@
 import unittest
 
 from transformers import VitDetConfig
-from transformers.testing_utils import require_torch, torch_device
+from transformers.testing_utils import is_flaky, require_torch, torch_device
 from transformers.utils import is_torch_available
 
 from ...test_backbone_common import BackboneTesterMixin
@@ -175,6 +175,10 @@ class VitDetModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
         self.model_tester = VitDetModelTester(self)
         self.config_tester = ConfigTester(self, config_class=VitDetConfig, has_text_modality=False, hidden_size=37)
 
+    @is_flaky(max_attempts=3, description="`torch.nn.init.trunc_normal_` is flaky.")
+    def test_initialization(self):
+        super().test_initialization()
+
     # TODO: Fix me (once this model gets more usage)
     @unittest.skip("Does not work on the tiny model as we keep hitting edge cases.")
     def test_cpu_offload(self):