Bugfix for SDXL model loading in low ram system. (#4628)

Update convert_from_ckpt.py
Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>

src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py

@@ -1387,6 +1387,7 @@ def download_from_original_stable_diffusion_ckpt(
         unet = UNet2DConditionModel(**unet_config)
 
     if is_accelerate_available():
+        if model_type not in ["SDXL", "SDXL-Refiner"]: # SBM Delay this.
             for param_name, param in converted_unet_checkpoint.items():
                 set_module_tensor_to_device(unet, param_name, "cpu", value=param)
     else:
@@ -1588,6 +1589,11 @@ def download_from_original_stable_diffusion_ckpt(
                 checkpoint, config_name, prefix="conditioner.embedders.1.model.", has_projection=True, **config_kwargs
             )
 
+            if is_accelerate_available(): # SBM Now move model to cpu.
+                if model_type in ["SDXL", "SDXL-Refiner"]:
+                    for param_name, param in converted_unet_checkpoint.items():
+                        set_module_tensor_to_device(unet, param_name, "cpu", value=param)
+
             pipe = pipeline_class(
                 vae=vae,
                 text_encoder=text_encoder,
@@ -1611,6 +1617,11 @@ def download_from_original_stable_diffusion_ckpt(
                 checkpoint, config_name, prefix="conditioner.embedders.0.model.", has_projection=True, **config_kwargs
             )
 
+            if is_accelerate_available(): # SBM Now move model to cpu.
+                if model_type in ["SDXL", "SDXL-Refiner"]:
+                    for param_name, param in converted_unet_checkpoint.items():
+                        set_module_tensor_to_device(unet, param_name, "cpu", value=param)
+            
             pipe = StableDiffusionXLImg2ImgPipeline(
                 vae=vae,
                 text_encoder=text_encoder,