support rebuild train loader when data has expired

Summary:
- store expiration in meta data when loading dat

- use before_train_hook to rebuild data loader when expiration condition is met.

Reviewed By: zisting

Differential Revision: D27683164

fbshipit-source-id: e3e3c6c15eee7c02c7a1bfed5f4d4d0e67d61a4f

d2go/runner/default_runner.py

@@ -438,6 +438,7 @@ class Detectron2GoRunner(BaseRunner):
         trainer_hooks = [
             hooks.IterationTimer(),
             model_ema.EMAHook(cfg, model) if cfg.MODEL_EMA.ENABLED else None,
+            self._create_data_loader_hook(cfg),
             self._create_after_step_hook(
                 cfg, model, optimizer, scheduler, periodic_checkpointer
             ),
@@ -553,6 +554,12 @@ class Detectron2GoRunner(BaseRunner):
 
         return hooks.CallbackHook(after_step=after_step_callback)
 
+    def _create_data_loader_hook(self, cfg):
+        """
+        Create a hook for manipulating data loader
+        """
+        return None
+
     def _create_qat_hook(self, cfg):
         """
         Create a hook to start QAT (during training) and/or change the phase of QAT.