[LoRA] improve failure handling for peft. (#10551)

* improve failure handling for peft.

* emppty

* Update src/diffusers/loaders/peft.py
Co-authored-by: Benjamin Bossan <BenjaminBossan@users.noreply.github.com>

---------
Co-authored-by: Benjamin Bossan <BenjaminBossan@users.noreply.github.com>

src/diffusers/loaders/peft.py

@@ -300,7 +300,9 @@ class PeftAdapterMixin:
             try:
                 inject_adapter_in_model(lora_config, self, adapter_name=adapter_name, **peft_kwargs)
                 incompatible_keys = set_peft_model_state_dict(self, state_dict, adapter_name, **peft_kwargs)
-            except RuntimeError as e:
+            except Exception as e:
+                # In case `inject_adapter_in_model()` was unsuccessful even before injecting the `peft_config`.
+                if hasattr(self, "peft_config"):
                     for module in self.modules():
                         if isinstance(module, BaseTunerLayer):
                             active_adapters = module.active_adapters