Enable configuration of async write metrics

Summary:
Pull Request resolved: https://github.com/facebookresearch/d2go/pull/529

Set config param for enabling async write metrics added in D44305165

Use it in LDM Pokemon config as first use case

Reviewed By: sf-wind

Differential Revision: D44335491

fbshipit-source-id: b000502e6ed0e19a10d6fe3a7470bcd3045e7717

d2go/runner/config_defaults.py

@@ -153,6 +153,8 @@ def get_base_runner_default_cfg(cfg: CN) -> CN:
     cfg.GATHER_METRIC_PERIOD = 1
     # Frequency of metric printer, tensorboard writer, etc.
     cfg.WRITER_PERIOD = 20
+    # Enable async writing metrics to tensorboard and logs to speed up training
+    cfg.ASYNC_WRITE_METRICS = False
 
     # train_net specific arguments, define in runner but used in train_net
     # run evaluation after training is done

d2go/runner/default_runner.py

@@ -554,6 +554,7 @@ class Detectron2GoRunner(D2GoDataAPIMixIn, BaseRunner):
                     cfg.SOLVER.AMP.PRECISION, lightning=False
                 ),
                 log_grad_scaler=cfg.SOLVER.AMP.LOG_GRAD_SCALER,
+                async_write_metrics=cfg.ASYNC_WRITE_METRICS,
             )
         else:
             trainer = SimpleTrainer(
@@ -562,6 +563,7 @@ class Detectron2GoRunner(D2GoDataAPIMixIn, BaseRunner):
                 optimizer,
                 gather_metric_period=cfg.GATHER_METRIC_PERIOD,
                 zero_grad_before_forward=cfg.ZERO_GRAD_BEFORE_FORWARD,
+                async_write_metrics=cfg.ASYNC_WRITE_METRICS,
             )
 
         if cfg.SOLVER.AMP.ENABLED and torch.cuda.is_available():