Exit early in load if no weights are in the sharded state dict (#18937)

src/transformers/modeling_utils.py

@@ -418,6 +418,9 @@ def _load_state_dict_into_model(model_to_load, state_dict, start_prefix):
     def load(module: nn.Module, state_dict, prefix=""):
         local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
         args = (state_dict, prefix, local_metadata, True, [], [], error_msgs)
+        # Parameters of module and children will start with prefix. We can exit early if there are none in this
+        # state_dict
+        if len([key for key in state_dict if key.startswith(prefix)]) > 0:
             if is_deepspeed_zero3_enabled():
                 import deepspeed