higher atol to avoid flaky trainer test failure (#17979)

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

tests/trainer/test_trainer.py

@@ -1252,8 +1252,8 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
             trainer.train(resume_from_checkpoint=os.path.join(tmp_dir, "checkpoint-15"))
             (a1, b1) = trainer.model.a.item(), trainer.model.b.item()
 
-            self.assertAlmostEqual(a, a1, delta=1e-8)
-            self.assertAlmostEqual(b, b1, delta=1e-8)
+            self.assertAlmostEqual(a, a1, delta=1e-5)
+            self.assertAlmostEqual(b, b1, delta=1e-5)
 
         with self.subTest("Test every epoch"):
             config = RegressionModelConfig(a=0, b=2, random_torch=random_torch)
@@ -1277,8 +1277,8 @@ class TrainerIntegrationTest(TestCasePlus, TrainerIntegrationCommon):
             trainer.train(resume_from_checkpoint=os.path.join(tmp_dir, checkpoint_dir))
             (a1, b1) = trainer.model.a.item(), trainer.model.b.item()
 
-            self.assertAlmostEqual(a, a1, delta=1e-8)
-            self.assertAlmostEqual(b, b1, delta=1e-8)
+            self.assertAlmostEqual(a, a1, delta=1e-5)
+            self.assertAlmostEqual(b, b1, delta=1e-5)
 
     @slow
     @require_torch_non_multi_gpu