safeguard torch distributed check (#26056)

73b13ac0 · Sourab Mangrulkar · GitHub · 12f043ea · 73b13ac0
Unverified Commit 73b13ac0 authored Sep 13, 2023 by Sourab Mangrulkar Committed by GitHub Sep 13, 2023
Show whitespace changes
Inline Side-by-side

Showing with 5 additions and 1 deletion

src/transformers/modeling_utils.py src/transformers/modeling_utils.py +5 -1

No files found.
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -120,7 +120,11 @@ _init_weights = True
 def is_fsdp_enabled():
-    return torch.distributed.is_initialized() and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
+    return (
+        torch.distributed.is_available()
+        and torch.distributed.is_initialized()
+        and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
+    )
 def is_fsdp_enabled_and_dist_rank_0():