Fix is_torch_xpu_available for torch < 2.3 (#31573)

src/transformers/utils/import_utils.py

@@ -754,10 +754,13 @@ def is_torch_xpu_available(check_device=False):
     if not is_torch_available():
         return False
 
-    import torch
-
+    torch_version = version.parse(_torch_version)
     if is_ipex_available():
         import intel_extension_for_pytorch  # noqa: F401
+    elif torch_version.major < 2 or (torch_version.major == 2 and torch_version.minor < 4):
+        return False
+
+    import torch
 
     if check_device:
         try: