disable memory profiler by default + remove force disable + add logging

Summary: Pull Request resolved: https://github.com/facebookresearch/d2go/pull/581

Reviewed By: wat3rBro

Differential Revision: D46913792

fbshipit-source-id: cf3c3812c455091fbf63842443644d2571976017

d2go/runner/default_runner.py

@@ -343,8 +343,7 @@ class Detectron2GoRunner(D2GoDataAPIMixIn, BaseRunner):
 
     def build_model(self, cfg, eval_only=False):
         # Attach memory profiler to GPU OOM events
-        # Disabled since it can cause ranks to die
-        if False and cfg.get("MEMORY_PROFILER", CfgNode()).get("ENABLED", False):
+        if cfg.get("MEMORY_PROFILER", CfgNode()).get("ENABLED", False):
             attach_oom_logger(
                 cfg.OUTPUT_DIR, trace_max_entries=cfg.MEMORY_PROFILER.TRACE_MAX_ENTRIES
             )

d2go/runner/training_hooks.py

@@ -51,6 +51,9 @@ class D2GoGpuMemorySnapshot(HookBase):
         self.log_n_steps = log_n_steps
         self.log_during_train_at = log_during_train_at
         self.trace_max_entries = trace_max_entries
+        logger.warning(
+            "WARNING: Memory snapshot profiler is enabled. This may cause ranks to die and training jobs to get stuck. Please use with caution."
+        )
 
     def before_step(self):
         if self.trainer.iter == self.log_during_train_at: