Fix lora device test (#7738)

* fix lora device test

* fix more.

* fix more/

* quality

* empty

---------
Co-authored-by: Dhruv Nair <dhruv.nair@gmail.com>

src/diffusers/loaders/lora.py

@@ -1268,6 +1268,7 @@ class LoraLoaderMixin:
                     unet_module.lora_A[adapter_name].to(device)
                     unet_module.lora_B[adapter_name].to(device)
                     # this is a param, not a module, so device placement is not in-place -> re-assign
+                    if hasattr(unet_module, "lora_magnitude_vector") and unet_module.lora_magnitude_vector is not None:
                         unet_module.lora_magnitude_vector[adapter_name] = unet_module.lora_magnitude_vector[
                             adapter_name
                         ].to(device)
@@ -1288,6 +1289,10 @@ class LoraLoaderMixin:
                         text_encoder_module.lora_A[adapter_name].to(device)
                         text_encoder_module.lora_B[adapter_name].to(device)
                         # this is a param, not a module, so device placement is not in-place -> re-assign
+                        if (
+                            hasattr(text_encoder, "lora_magnitude_vector")
+                            and text_encoder_module.lora_magnitude_vector is not None
+                        ):
                             text_encoder_module.lora_magnitude_vector[
                                 adapter_name
                             ] = text_encoder_module.lora_magnitude_vector[adapter_name].to(device)