[Core] fix offload behaviour when device_map is enabled. (#7919)

fix offload behaviour when device_map is enabled.

src/diffusers/loaders/lora.py

@@ -363,7 +363,7 @@ class LoraLoaderMixin:
         is_model_cpu_offload = False
         is_sequential_cpu_offload = False
 
-        if _pipeline is not None:
+        if _pipeline is not None and _pipeline.hf_device_map is None:
             for _, component in _pipeline.components.items():
                 if isinstance(component, nn.Module) and hasattr(component, "_hf_hook"):
                     if not is_model_cpu_offload:

src/diffusers/loaders/textual_inversion.py

@@ -419,6 +419,7 @@ class TextualInversionLoaderMixin:
         # 7.1 Offload all hooks in case the pipeline was cpu offloaded before make sure, we offload and onload again
         is_model_cpu_offload = False
         is_sequential_cpu_offload = False
+        if self.hf_device_map is None:
             for _, component in self.components.items():
                 if isinstance(component, nn.Module):
                     if hasattr(component, "_hf_hook"):