fix deepspeed available detection (#26252)

src/transformers/integrations/deepspeed.py

@@ -15,6 +15,7 @@
 Integration with Deepspeed
 """
 
+import importlib.metadata as importlib_metadata
 import importlib.util
 import weakref
 from functools import partialmethod
@@ -32,7 +33,16 @@ logger = logging.get_logger(__name__)
 
 
 def is_deepspeed_available():
-    return importlib.util.find_spec("deepspeed") is not None
+    package_exists = importlib.util.find_spec("deepspeed") is not None
+
+    # Check we're not importing a "deepspeed" directory somewhere but the actual library by trying to grab the version
+    # AND checking it has an author field in the metadata that is HuggingFace.
+    if package_exists:
+        try:
+            _ = importlib_metadata.metadata("deepspeed")
+            return True
+        except importlib_metadata.PackageNotFoundError:
+            return False
 
 
 if is_accelerate_available() and is_deepspeed_available():