Faster set_adapters (#10777)

* Update peft_utils.py

* Update peft_utils.py

* Update peft_utils.py

---------
Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>

src/diffusers/utils/peft_utils.py

@@ -257,26 +257,18 @@ def set_weights_and_activate_adapters(model, adapter_names, weights):
 
         return block_weight
 
-    # iterate over each adapter, make it active and set the corresponding scaling weight
-    for adapter_name, weight in zip(adapter_names, weights):
-        for module_name, module in model.named_modules():
-            if isinstance(module, BaseTunerLayer):
-                # For backward compatbility with previous PEFT versions
-                if hasattr(module, "set_adapter"):
-                    module.set_adapter(adapter_name)
-                else:
-                    module.active_adapter = adapter_name
-                module.set_scale(adapter_name, get_module_weight(weight, module_name))
-
-    # set multiple active adapters
-    for module in model.modules():
+    for module_name, module in model.named_modules():
         if isinstance(module, BaseTunerLayer):
-            # For backward compatbility with previous PEFT versions
+            # For backward compatibility with previous PEFT versions, set multiple active adapters
             if hasattr(module, "set_adapter"):
                 module.set_adapter(adapter_names)
             else:
                 module.active_adapter = adapter_names
 
+            # Set the scaling weight for each adapter for this module
+            for adapter_name, weight in zip(adapter_names, weights):
+                module.set_scale(adapter_name, get_module_weight(weight, module_name))
+
 
 def check_peft_version(min_version: str) -> None:
     r"""