Fix case when using --gradient_accumulation_steps with DDP disabled. (#22007)

Co-authored-by: EC2 Default User <ec2-user@ip-172-31-42-72.us-west-2.compute.internal>

src/transformers/training_args.py

@@ -1801,7 +1801,9 @@ class TrainingArguments:
         """
         Whether or not to use no_sync for the gradients when doing gradient accumulation.
         """
-        return not (self.deepspeed or is_sagemaker_dp_enabled() or is_sagemaker_mp_enabled())
+        return not (
+            self.deepspeed or is_sagemaker_dp_enabled() or is_sagemaker_mp_enabled() or is_torch_neuroncore_available()
+        )
 
     @contextlib.contextmanager
     def main_process_first(self, local=True, desc="work"):