Fix if else and *actually* enable superfast init (#32007)

* Fix if else

* rm err raise

src/transformers/modeling_utils.py

@@ -349,7 +349,7 @@ def check_support_param_buffer_assignment(model_to_load, state_dict, start_prefi
         return False
 
     # Some models explicitly do not support param buffer assignment
-    if not getattr(model_to_load, "_supports_param_buffer_assignment", False):
+    if not getattr(model_to_load, "_supports_param_buffer_assignment", True):
         logger.debug(
             f"{model_to_load.__class__.__name__} does not support param buffer assignment, loading will be slower"
         )