adding training deterministic setups

Summary: Pull Request resolved: https://github.com/facebookresearch/d2go/pull/617

Reviewed By: tglik

Differential Revision:
D49065205

Privacy Context Container: L1181999

fbshipit-source-id: b8e8b994a2bd32967dbb9afbc0d8fcfa7ef59667

d2go/runner/default_runner.py

@@ -618,6 +618,9 @@ class Detectron2GoRunner(D2GoDataAPIMixIn, BaseRunner):
                 # For more details https://pytorch.org/docs/stable/notes/cuda.html#tf32-on-ampere.
                 torch.backends.cuda.matmul.allow_tf32 = True
                 torch.backends.cudnn.allow_tf32 = True
+            elif cfg.SOLVER.DETERMINISTIC:
+                torch.backends.cuda.matmul.allow_tf32 = False
+                torch.backends.cudnn.allow_tf32 = False
 
             trainer_hooks = self._get_trainer_hooks(
                 cfg, model, optimizer, scheduler, periodic_checkpointer, trainer

d2go/setup.py

@@ -324,6 +324,7 @@ def setup_after_launch(
     # avoid random pytorch and CUDA algorithms during the training
     if cfg.SOLVER.DETERMINISTIC:
         logging.warning("Using deterministic training for the reproducibility")
+        torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False
         torch.use_deterministic_algorithms(True)
         # reference: https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility