Allow controlnets to be loaded (from ckpt) in a parallel thread with a SD...

Allow controlnets to be loaded (from ckpt) in a parallel thread with a SD model (ckpt), and speed it up slightly (#4298)

Faster controlnet model instantiation, and allow controlnets to be loaded (from ckpt) in a parallel thread with a SD model (ckpt) without  tensor errors (race condition)

src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py

@@ -1079,6 +1079,8 @@ def convert_controlnet_checkpoint(
     if cross_attention_dim is not None:
         ctrlnet_config["cross_attention_dim"] = cross_attention_dim
 
+    ctx = init_empty_weights if is_accelerate_available() else nullcontext
+    with ctx():
         controlnet = ControlNetModel(**ctrlnet_config)
 
     # Some controlnet ckpt files are distributed independently from the rest of the
@@ -1097,6 +1099,10 @@ def convert_controlnet_checkpoint(
         skip_extract_state_dict=skip_extract_state_dict,
     )
 
+    if is_accelerate_available():
+        for param_name, param in converted_ctrl_checkpoint.items():
+            set_module_tensor_to_device(controlnet, param_name, "cpu", value=param)
+    else:
         controlnet.load_state_dict(converted_ctrl_checkpoint)
 
     return controlnet