Fix/controlnet lora (#5157)

* print

* print

* print

* print

* print

* debugging

* debugging

* debugging

* debugging

* safer condition.

* remove prints and try excepts.

* Empty-Commit

* Apply suggestions from code review

---------
Co-authored-by: Patrick von Platen <patrick.v.platen@gmail.com>

src/diffusers/loaders.py

@@ -1922,7 +1922,7 @@ class LoraLoaderMixin:
                 diffusers_name = diffusers_name.replace("emb.layers", "time_emb_proj")
 
                 # SDXL specificity.
-                if "emb" in diffusers_name and "time" not in diffusers_name:
+                if "emb" in diffusers_name and "time.emb.proj" not in diffusers_name:
                     pattern = r"\.\d+(?=\D*$)"
                     diffusers_name = re.sub(pattern, "", diffusers_name, count=1)
                 if ".in." in diffusers_name:
@@ -1935,7 +1935,7 @@ class LoraLoaderMixin:
                     diffusers_name = diffusers_name.replace("skip.connection", "conv_shortcut")
 
                 # LyCORIS specificity.
-                if "time" in diffusers_name:
+                if "time.emb.proj" in diffusers_name:
                     diffusers_name = diffusers_name.replace("time.emb.proj", "time_emb_proj")
                 if "conv.shortcut" in diffusers_name:
                     diffusers_name = diffusers_name.replace("conv.shortcut", "conv_shortcut")