Fix common tests for FP16 (#1588)

* Fix common tests for FP16

* revert

tests/pipelines/ddim/test_ddim.py

@@ -29,6 +29,7 @@ torch.backends.cuda.matmul.allow_tf32 = False
 
 class DDIMPipelineFastTests(PipelineTesterMixin, unittest.TestCase):
     pipeline_class = DDIMPipeline
+    test_cpu_offload = False
 
     def get_dummy_components(self):
         torch.manual_seed(0)

tests/test_pipelines_common.py

@@ -190,7 +190,7 @@ class PipelineTesterMixin:
 
         with tempfile.TemporaryDirectory() as tmpdir:
             pipe.save_pretrained(tmpdir)
-            pipe_loaded = self.pipeline_class.from_pretrained(tmpdir)
+            pipe_loaded = self.pipeline_class.from_pretrained(tmpdir, torch_dtype=torch.float16)
             pipe_loaded.to(torch_device)
             pipe_loaded.set_progress_bar_config(disable=None)