Fix apex Adam to not break CPU mode

Reviewed By: chenyangyu1988

Differential Revision: D14784219

fbshipit-source-id: 273888d6e3d22a01d5e7edfbc786195e7b78efef

fairseq/optim/adam.py

@@ -14,12 +14,16 @@ from . import FairseqOptimizer, register_optimizer
 
 @register_optimizer('adam')
 class FairseqAdam(FairseqOptimizer):
+
     def __init__(self, args, params):
         super().__init__(args, params)
-        try:
-            from apex.optimizers import FusedAdam
-            self._optimizer = FusedAdam(params, **self.optimizer_config)
-        except ImportError:
+        if torch.cuda.is_available():
+            try:
+                from apex.optimizers import FusedAdam
+                self._optimizer = FusedAdam(params, **self.optimizer_config)
+            except ImportError:
+                self._optimizer = Adam(params, **self.optimizer_config)
+        else:
             self._optimizer = Adam(params, **self.optimizer_config)
 
     @staticmethod